Compare commits

...

22 Commits

Author SHA1 Message Date
nub31
c3d64c4ea9 Update lsp to have set root path 2025-11-05 15:53:08 +01:00
nub31
d3822bc9b4 ... 2025-11-05 15:20:45 +01:00
nub31
36622755a9 ... 2025-11-03 19:54:41 +01:00
nub31
47fef6bc9f ... 2025-11-03 17:10:15 +01:00
nub31
7d49bf43b7 ... 2025-11-03 16:01:20 +01:00
nub31
7ce451768d reformat generator 2025-11-03 14:02:47 +01:00
nub31
f231a45285 ... 2025-11-03 13:46:25 +01:00
nub31
085f7a1a6a ... 2025-11-03 12:52:17 +01:00
nub31
40d500fddd ... 2025-10-31 15:18:18 +01:00
nub31
7c7624b1bc ... 2025-10-31 14:42:58 +01:00
nub31
031b118a24 ... 2025-10-31 11:59:53 +01:00
nub31
c764857561 ... 2025-10-29 18:41:52 +01:00
nub31
4f724ddc0c remove refs 2025-10-29 17:45:25 +01:00
nub31
bf4c8838c6 ... 2025-10-29 15:14:13 +01:00
nub31
34a44f80a8 ... 2025-10-27 15:36:03 +01:00
nub31
640bd8c573 ... 2025-10-27 15:33:44 +01:00
nub31
560e6428ff ... 2025-10-26 22:28:48 +01:00
nub31
27bc4da4fd ... 2025-10-26 20:04:57 +01:00
nub31
d11df414ad Refs working? 2025-10-26 17:05:56 +01:00
nub31
3febaaea81 ref 2025-10-26 16:07:49 +01:00
nub31
828e20ddb6 ... 2025-10-26 14:15:25 +01:00
nub31
396ddf93a2 ... 2025-10-25 18:07:34 +02:00
41 changed files with 2989 additions and 2045 deletions

1
.gitignore vendored Normal file
View File

@@ -0,0 +1 @@
.vscode

View File

@@ -1,32 +1,57 @@
using System.Diagnostics; using NubLang.Ast;
using NubLang.Ast;
using NubLang.Diagnostics; using NubLang.Diagnostics;
using NubLang.Generation; using NubLang.Generation;
using NubLang.Modules;
using NubLang.Syntax; using NubLang.Syntax;
var diagnostics = new List<Diagnostic>(); var diagnostics = new List<Diagnostic>();
var syntaxTrees = new List<SyntaxTree>(); var syntaxTrees = new List<SyntaxTree>();
var tokenizer = new Tokenizer();
var parser = new Parser();
var generator = new LlvmSharpGenerator();
foreach (var file in args) foreach (var file in args)
{ {
var tokenizer = new Tokenizer(file, File.ReadAllText(file)); var tokens = tokenizer.Tokenize(file, File.ReadAllText(file));
tokenizer.Tokenize();
diagnostics.AddRange(tokenizer.Diagnostics); diagnostics.AddRange(tokenizer.Diagnostics);
var parser = new Parser(); var syntaxTree = parser.Parse(tokens);
var syntaxTree = parser.Parse(tokenizer.Tokens);
diagnostics.AddRange(parser.Diagnostics); diagnostics.AddRange(parser.Diagnostics);
syntaxTrees.Add(syntaxTree); syntaxTrees.Add(syntaxTree);
} }
var modules = Module.Collect(syntaxTrees); foreach (var diagnostic in diagnostics)
var compilationUnits = new List<CompilationUnit>(); {
Console.Error.WriteLine(diagnostic.FormatANSI());
}
if (diagnostics.Any(diagnostic => diagnostic.Severity == DiagnosticSeverity.Error))
{
return 1;
}
diagnostics.Clear();
ModuleRepository moduleRepository;
try
{
moduleRepository = ModuleRepository.Create(syntaxTrees);
}
catch (CompileException e)
{
Console.Error.WriteLine(e.Diagnostic.FormatANSI());
return 1;
}
var compilationUnits = new List<List<TopLevelNode>>();
for (var i = 0; i < args.Length; i++) for (var i = 0; i < args.Length; i++)
{ {
var typeChecker = new TypeChecker(syntaxTrees[i], modules); var typeChecker = new TypeChecker();
var compilationUnit = typeChecker.Check(); var compilationUnit = typeChecker.Check(syntaxTrees[i], moduleRepository);
compilationUnits.Add(compilationUnit); compilationUnits.Add(compilationUnit);
diagnostics.AddRange(typeChecker.Diagnostics); diagnostics.AddRange(typeChecker.Diagnostics);
@@ -42,7 +67,7 @@ if (diagnostics.Any(diagnostic => diagnostic.Severity == DiagnosticSeverity.Erro
return 1; return 1;
} }
var cPaths = new List<string>(); diagnostics.Clear();
Directory.CreateDirectory(".build"); Directory.CreateDirectory(".build");
@@ -51,41 +76,14 @@ for (var i = 0; i < args.Length; i++)
var file = args[i]; var file = args[i];
var compilationUnit = compilationUnits[i]; var compilationUnit = compilationUnits[i];
var generator = new Generator(compilationUnit);
var directory = Path.GetDirectoryName(file); var directory = Path.GetDirectoryName(file);
if (!string.IsNullOrWhiteSpace(directory)) if (!string.IsNullOrWhiteSpace(directory))
{ {
Directory.CreateDirectory(Path.Combine(".build", directory)); Directory.CreateDirectory(Path.Combine(".build", directory));
} }
var path = Path.Combine(".build", Path.ChangeExtension(file, "c")); var path = Path.Combine(".build", Path.ChangeExtension(file, "ll"));
File.WriteAllText(path, generator.Emit()); generator.Emit(compilationUnit, moduleRepository, file, path);
cPaths.Add(path);
} }
var objectPaths = new List<string>();
foreach (var cPath in cPaths)
{
var objectPath = Path.ChangeExtension(cPath, "o");
using var compileProcess = Process.Start("clang", [
"-ffreestanding", "-std=c23",
"-g", "-c",
"-o", objectPath,
cPath,
]);
compileProcess.WaitForExit();
if (compileProcess.ExitCode != 0)
{
Console.Error.WriteLine($"clang failed with exit code {compileProcess.ExitCode}");
return 1;
}
objectPaths.Add(objectPath);
}
Console.Out.WriteLine(string.Join(' ', objectPaths));
return 0; return 0;

View File

@@ -16,25 +16,26 @@ public static class AstExtensions
return new Location return new Location
{ {
Uri = node.Tokens.First().Span.FilePath, Uri = node.Tokens.First().Span.SourcePath,
Range = new Range(node.Tokens.First().Span.Start.Line - 1, node.Tokens.First().Span.Start.Column - 1, node.Tokens.Last().Span.End.Line - 1, node.Tokens.Last().Span.End.Column - 1) Range = new Range(node.Tokens.First().Span.StartLine - 1, node.Tokens.First().Span.StartColumn - 1, node.Tokens.Last().Span.EndLine - 1, node.Tokens.Last().Span.EndColumn - 1)
}; };
} }
public static bool ContainsPosition(this Node node, int line, int character) public static Location ToLocation(this Token token)
{ {
if (node.Tokens.Count == 0) return new Location
{ {
return false; Uri = token.Span.SourcePath,
} Range = new Range(token.Span.StartLine - 1, token.Span.StartColumn - 1, token.Span.EndLine - 1, token.Span.EndColumn - 1)
};
}
var start = node.Tokens.First().Span.Start; public static bool ContainsPosition(this Token token, int line, int character)
var end = node.Tokens.Last().Span.End; {
var startLine = token.Span.StartLine - 1;
var startLine = start.Line - 1; var startChar = token.Span.StartColumn - 1;
var startChar = start.Column - 1; var endLine = token.Span.EndLine - 1;
var endLine = end.Line - 1; var endChar = token.Span.EndColumn - 1;
var endChar = end.Column - 1;
if (line < startLine || line > endLine) return false; if (line < startLine || line > endLine) return false;
@@ -58,20 +59,56 @@ public static class AstExtensions
return false; return false;
} }
public static FuncNode? FunctionAtPosition(this CompilationUnit compilationUnit, int line, int character) public static bool ContainsPosition(this Node node, int line, int character)
{
if (node.Tokens.Count == 0)
{
return false;
}
var span = node.Tokens.First().Span;
var startLine = span.StartLine - 1;
var startChar = span.StartColumn - 1;
var endLine = span.EndLine - 1;
var endChar = span.EndColumn - 1;
if (line < startLine || line > endLine) return false;
if (line > startLine && line < endLine) return true;
if (startLine == endLine)
{
return character >= startChar && character <= endChar;
}
if (line == startLine)
{
return character >= startChar;
}
if (line == endLine)
{
return character <= endChar;
}
return false;
}
public static FuncNode? FunctionAtPosition(this List<TopLevelNode> compilationUnit, int line, int character)
{ {
return compilationUnit return compilationUnit
.Functions .OfType<FuncNode>()
.FirstOrDefault(x => x.ContainsPosition(line, character)); .FirstOrDefault(x => x.ContainsPosition(line, character));
} }
public static Node? DeepestNodeAtPosition(this CompilationUnit compilationUnit, int line, int character) public static Node? DeepestNodeAtPosition(this List<TopLevelNode> compilationUnit, int line, int character)
{ {
return compilationUnit.Functions return compilationUnit
.SelectMany(x => x.DescendantsAndSelf()) .SelectMany(x => x.DescendantsAndSelf())
.Where(n => n.ContainsPosition(line, character)) .Where(n => n.ContainsPosition(line, character))
.OrderBy(n => n.Tokens.First().Span.Start.Line) .OrderBy(n => n.Tokens.First().Span.StartLine)
.ThenBy(n => n.Tokens.First().Span.Start.Column) .ThenBy(n => n.Tokens.First().Span.StartColumn)
.LastOrDefault(); .LastOrDefault();
} }
} }

View File

@@ -1,4 +1,5 @@
using NubLang.Ast; using NubLang.Ast;
using NubLang.Modules;
using OmniSharp.Extensions.LanguageServer.Protocol.Client.Capabilities; using OmniSharp.Extensions.LanguageServer.Protocol.Client.Capabilities;
using OmniSharp.Extensions.LanguageServer.Protocol.Document; using OmniSharp.Extensions.LanguageServer.Protocol.Document;
using OmniSharp.Extensions.LanguageServer.Protocol.Models; using OmniSharp.Extensions.LanguageServer.Protocol.Models;
@@ -29,13 +30,6 @@ internal class CompletionHandler(WorkspaceManager workspaceManager) : Completion
Label = "module", Label = "module",
InsertTextFormat = InsertTextFormat.Snippet, InsertTextFormat = InsertTextFormat.Snippet,
InsertText = "module \"$0\"", InsertText = "module \"$0\"",
},
new()
{
Kind = CompletionItemKind.Keyword,
Label = "import",
InsertTextFormat = InsertTextFormat.Snippet,
InsertText = "import \"$0\"",
} }
]; ];
@@ -112,62 +106,76 @@ internal class CompletionHandler(WorkspaceManager workspaceManager) : Completion
private CompletionList HandleSync(CompletionParams request, CancellationToken cancellationToken) private CompletionList HandleSync(CompletionParams request, CancellationToken cancellationToken)
{ {
var completions = new List<CompletionItem>(); var completions = new List<CompletionItem>();
var position = request.Position;
var uri = request.TextDocument.Uri; var compilationUnit = workspaceManager.GetTopLevelNodes(request.TextDocument.Uri.GetFileSystemPath());
var compilationUnit = workspaceManager.GetCompilationUnit(uri);
if (compilationUnit != null) var repository = workspaceManager.GetModuleRepository();
var function = compilationUnit
.OfType<FuncNode>()
.FirstOrDefault(x => x.Body != null && x.Body.ContainsPosition(request.Position.Line, request.Position.Character));
if (function != null)
{ {
var function = compilationUnit.Functions.FirstOrDefault(x => x.Body != null && x.Body.ContainsPosition(position.Line, position.Character)); completions.AddRange(_statementSnippets);
if (function != null)
{
completions.AddRange(_statementSnippets);
foreach (var prototype in compilationUnit.ImportedFunctions) foreach (var module in repository.GetAll())
{
foreach (var prototype in module.FunctionPrototypes)
{ {
var parameterStrings = new List<string>(); var parameterStrings = new List<string>();
foreach (var (index, parameter) in prototype.Parameters.Index()) foreach (var (index, parameter) in prototype.Parameters.Index())
{ {
parameterStrings.AddRange($"${{{index + 1}:{parameter.Name}}}"); parameterStrings.AddRange($"${{{index + 1}:{parameter.NameToken.Value}}}");
}
var isCurrentModule = false;
var moduleDecl = compilationUnit.OfType<ModuleNode>().FirstOrDefault();
if (moduleDecl != null)
{
if (moduleDecl.NameToken.Value == module.Name)
{
isCurrentModule = true;
}
} }
completions.Add(new CompletionItem completions.Add(new CompletionItem
{ {
Kind = CompletionItemKind.Function, Kind = CompletionItemKind.Function,
Label = $"{prototype.Module}::{prototype.Name}", Label = isCurrentModule ? prototype.NameToken.Value : $"{module.Name}::{prototype.NameToken.Value}",
InsertTextFormat = InsertTextFormat.Snippet, InsertTextFormat = InsertTextFormat.Snippet,
InsertText = $"{prototype.Module}::{prototype.Name}({string.Join(", ", parameterStrings)})", InsertText = $"{(isCurrentModule ? "" : $"{module.Name}::")}{prototype.NameToken.Value}({string.Join(", ", parameterStrings)})",
});
}
foreach (var parameter in function.Prototype.Parameters)
{
completions.Add(new CompletionItem
{
Kind = CompletionItemKind.Variable,
Label = parameter.Name,
InsertText = parameter.Name,
});
}
var variables = function.Body!
.Descendants()
.OfType<VariableDeclarationNode>();
foreach (var variable in variables)
{
completions.Add(new CompletionItem
{
Kind = CompletionItemKind.Variable,
Label = variable.Name,
InsertText = variable.Name,
}); });
} }
} }
else
foreach (var parameter in function.Prototype.Parameters)
{ {
completions.AddRange(_definitionSnippets); completions.Add(new CompletionItem
{
Kind = CompletionItemKind.Variable,
Label = parameter.NameToken.Value,
InsertText = parameter.NameToken.Value,
});
} }
var variables = function.Body!
.Descendants()
.OfType<VariableDeclarationNode>();
foreach (var variable in variables)
{
completions.Add(new CompletionItem
{
Kind = CompletionItemKind.Variable,
Label = variable.NameToken.Value,
InsertText = variable.NameToken.Value,
});
}
}
else
{
completions.AddRange(_definitionSnippets);
} }
return new CompletionList(completions, false); return new CompletionList(completions, false);

View File

@@ -20,47 +20,58 @@ internal class DefinitionHandler(WorkspaceManager workspaceManager) : Definition
private LocationOrLocationLinks? HandleSync(DefinitionParams request, CancellationToken cancellationToken) private LocationOrLocationLinks? HandleSync(DefinitionParams request, CancellationToken cancellationToken)
{ {
var uri = request.TextDocument.Uri; var uri = request.TextDocument.Uri;
var compilationUnit = workspaceManager.GetCompilationUnit(uri); var topLevelNodes = workspaceManager.GetTopLevelNodes(uri.GetFileSystemPath());
if (compilationUnit == null)
{
return null;
}
var line = request.Position.Line; var line = request.Position.Line;
var character = request.Position.Character; var character = request.Position.Character;
var node = compilationUnit.DeepestNodeAtPosition(line, character); var node = topLevelNodes.DeepestNodeAtPosition(line, character);
switch (node) switch (node)
{ {
case VariableIdentifierNode variableIdentifierNode: case VariableIdentifierNode variableIdentifierNode:
{ {
var function = compilationUnit.FunctionAtPosition(line, character); var funcNode = topLevelNodes.FunctionAtPosition(line, character);
var parameter = function?.Prototype.Parameters.FirstOrDefault(x => x.Name == variableIdentifierNode.Name); var parameter = funcNode?.Prototype.Parameters.FirstOrDefault(x => x.NameToken.Value == variableIdentifierNode.NameToken.Value);
if (parameter != null) if (parameter != null)
{ {
return new LocationOrLocationLinks(parameter.ToLocation()); return new LocationOrLocationLinks(parameter.NameToken.ToLocation());
} }
var variable = function?.Body? var variable = funcNode?.Body?
.Descendants() .Descendants()
.OfType<VariableDeclarationNode>() .OfType<VariableDeclarationNode>()
.FirstOrDefault(x => x.Name == variableIdentifierNode.Name); .FirstOrDefault(x => x.NameToken.Value == variableIdentifierNode.NameToken.Value);
if (variable != null) if (variable != null)
{ {
return new LocationOrLocationLinks(variable.ToLocation()); return new LocationOrLocationLinks(variable.NameToken.ToLocation());
} }
return null; return null;
} }
case FuncIdentifierNode funcIdentifierNode: case LocalFuncIdentifierNode localFuncIdentifierNode:
{ {
var prototype = compilationUnit.ImportedFunctions.FirstOrDefault(x => x.Module == funcIdentifierNode.Module && x.Name == funcIdentifierNode.Name); var funcNode = topLevelNodes.OfType<FuncNode>().FirstOrDefault(x => x.NameToken.Value == localFuncIdentifierNode.NameToken.Value);
if (prototype != null) if (funcNode != null)
{ {
return new LocationOrLocationLinks(prototype.ToLocation()); return new LocationOrLocationLinks(funcNode.NameToken.ToLocation());
}
return null;
}
case ModuleFuncIdentifierNode localFuncIdentifierNode:
{
var repository = workspaceManager.GetModuleRepository();
if (!repository.TryGet(localFuncIdentifierNode.ModuleToken, out var module))
{
return null;
}
if (module.TryResolveFunc(localFuncIdentifierNode.NameToken, out var func, out _))
{
return new LocationOrLocationLinks(func.NameToken.ToLocation());
} }
return null; return null;

View File

@@ -37,7 +37,7 @@ public class DiagnosticsPublisher
}, },
Message = $"{nubDiagnostic.Message}\n{(nubDiagnostic.Help == null ? "" : $"help: {nubDiagnostic.Help}")}", Message = $"{nubDiagnostic.Message}\n{(nubDiagnostic.Help == null ? "" : $"help: {nubDiagnostic.Help}")}",
Range = nubDiagnostic.Span.HasValue Range = nubDiagnostic.Span.HasValue
? new Range(nubDiagnostic.Span.Value.Start.Line - 1, nubDiagnostic.Span.Value.Start.Column - 1, nubDiagnostic.Span.Value.End.Line - 1, nubDiagnostic.Span.Value.End.Column - 1) ? new Range(nubDiagnostic.Span.Value.StartLine - 1, nubDiagnostic.Span.Value.StartColumn - 1, nubDiagnostic.Span.Value.EndLine - 1, nubDiagnostic.Span.Value.EndColumn - 1)
: new Range(), : new Range(),
}; };
} }

View File

@@ -1,5 +1,7 @@
using System.Globalization; using System.Globalization;
using NubLang.Ast; using NubLang.Ast;
using NubLang.Modules;
using NubLang.Types;
using OmniSharp.Extensions.LanguageServer.Protocol.Client.Capabilities; using OmniSharp.Extensions.LanguageServer.Protocol.Client.Capabilities;
using OmniSharp.Extensions.LanguageServer.Protocol.Document; using OmniSharp.Extensions.LanguageServer.Protocol.Document;
using OmniSharp.Extensions.LanguageServer.Protocol.Models; using OmniSharp.Extensions.LanguageServer.Protocol.Models;
@@ -23,8 +25,16 @@ internal class HoverHandler(WorkspaceManager workspaceManager) : HoverHandlerBas
private Hover? HandleSync(HoverParams request, CancellationToken cancellationToken) private Hover? HandleSync(HoverParams request, CancellationToken cancellationToken)
{ {
var compilationUnit = workspaceManager.GetCompilationUnit(request.TextDocument.Uri); var topLevelNodes = workspaceManager.GetTopLevelNodes(request.TextDocument.Uri.GetFileSystemPath());
if (compilationUnit == null)
var moduleDecl = topLevelNodes.OfType<ModuleNode>().FirstOrDefault();
if (moduleDecl == null)
{
return null;
}
var moduleRepository = workspaceManager.GetModuleRepository();
if (!moduleRepository.TryGet(moduleDecl.NameToken, out var module))
{ {
return null; return null;
} }
@@ -32,14 +42,14 @@ internal class HoverHandler(WorkspaceManager workspaceManager) : HoverHandlerBas
var line = request.Position.Line; var line = request.Position.Line;
var character = request.Position.Character; var character = request.Position.Character;
var hoveredNode = compilationUnit.DeepestNodeAtPosition(line, character); var hoveredNode = topLevelNodes.DeepestNodeAtPosition(line, character);
if (hoveredNode == null) if (hoveredNode == null)
{ {
return null; return null;
} }
var message = CreateMessage(hoveredNode, compilationUnit); var message = CreateMessage(hoveredNode, moduleRepository, module, line, character);
if (message == null) if (message == null)
{ {
return null; return null;
@@ -55,17 +65,18 @@ internal class HoverHandler(WorkspaceManager workspaceManager) : HoverHandlerBas
}; };
} }
private static string? CreateMessage(Node hoveredNode, CompilationUnit compilationUnit) private static string? CreateMessage(Node hoveredNode, ModuleRepository repository, ModuleRepository.Module currentModule, int line, int character)
{ {
return hoveredNode switch return hoveredNode switch
{ {
FuncNode funcNode => CreateFuncPrototypeMessage(funcNode.Prototype), FuncNode funcNode => CreateFuncPrototypeMessage(funcNode.Prototype),
FuncPrototypeNode funcPrototypeNode => CreateFuncPrototypeMessage(funcPrototypeNode), FuncPrototypeNode funcPrototypeNode => CreateFuncPrototypeMessage(funcPrototypeNode),
FuncIdentifierNode funcIdentifierNode => CreateFuncIdentifierMessage(funcIdentifierNode, compilationUnit), LocalFuncIdentifierNode funcIdentifierNode => CreateLocalFuncIdentifierMessage(funcIdentifierNode, currentModule),
FuncParameterNode funcParameterNode => CreateTypeNameMessage("Function parameter", funcParameterNode.Name, funcParameterNode.Type), ModuleFuncIdentifierNode funcIdentifierNode => CreateModuleFuncIdentifierMessage(funcIdentifierNode, repository),
VariableIdentifierNode variableIdentifierNode => CreateTypeNameMessage("Variable", variableIdentifierNode.Name, variableIdentifierNode.Type), FuncParameterNode funcParameterNode => CreateTypeNameMessage("Function parameter", funcParameterNode.NameToken.Value, funcParameterNode.Type),
VariableDeclarationNode variableDeclarationNode => CreateTypeNameMessage("Variable declaration", variableDeclarationNode.Name, variableDeclarationNode.Type), VariableIdentifierNode variableIdentifierNode => CreateTypeNameMessage("Variable", variableIdentifierNode.NameToken.Value, variableIdentifierNode.Type),
StructFieldAccessNode structFieldAccessNode => CreateTypeNameMessage("Struct field", $"{structFieldAccessNode.Target.Type}.{structFieldAccessNode.Field}", structFieldAccessNode.Type), VariableDeclarationNode variableDeclarationNode => CreateTypeNameMessage("Variable declaration", variableDeclarationNode.NameToken.Value, variableDeclarationNode.Type),
StructFieldAccessNode structFieldAccessNode => CreateTypeNameMessage("Struct field", $"{structFieldAccessNode.Target.Type}.{structFieldAccessNode.FieldToken.Value}", structFieldAccessNode.Type),
CStringLiteralNode cStringLiteralNode => CreateLiteralMessage(cStringLiteralNode.Type, '"' + cStringLiteralNode.Value + '"'), CStringLiteralNode cStringLiteralNode => CreateLiteralMessage(cStringLiteralNode.Type, '"' + cStringLiteralNode.Value + '"'),
StringLiteralNode stringLiteralNode => CreateLiteralMessage(stringLiteralNode.Type, '"' + stringLiteralNode.Value + '"'), StringLiteralNode stringLiteralNode => CreateLiteralMessage(stringLiteralNode.Type, '"' + stringLiteralNode.Value + '"'),
BoolLiteralNode boolLiteralNode => CreateLiteralMessage(boolLiteralNode.Type, boolLiteralNode.Value.ToString()), BoolLiteralNode boolLiteralNode => CreateLiteralMessage(boolLiteralNode.Type, boolLiteralNode.Value.ToString()),
@@ -79,6 +90,7 @@ internal class HoverHandler(WorkspaceManager workspaceManager) : HoverHandlerBas
U16LiteralNode u16LiteralNode => CreateLiteralMessage(u16LiteralNode.Type, u16LiteralNode.Value.ToString()), U16LiteralNode u16LiteralNode => CreateLiteralMessage(u16LiteralNode.Type, u16LiteralNode.Value.ToString()),
U32LiteralNode u32LiteralNode => CreateLiteralMessage(u32LiteralNode.Type, u32LiteralNode.Value.ToString()), U32LiteralNode u32LiteralNode => CreateLiteralMessage(u32LiteralNode.Type, u32LiteralNode.Value.ToString()),
U64LiteralNode u64LiteralNode => CreateLiteralMessage(u64LiteralNode.Type, u64LiteralNode.Value.ToString()), U64LiteralNode u64LiteralNode => CreateLiteralMessage(u64LiteralNode.Type, u64LiteralNode.Value.ToString()),
StructInitializerNode structInitializerNode => CreateStructInitializerMessage(structInitializerNode, line, character),
// Expressions can have a generic fallback showing the resulting type // Expressions can have a generic fallback showing the resulting type
ExpressionNode expressionNode => $""" ExpressionNode expressionNode => $"""
**Expression** `{expressionNode.GetType().Name}` **Expression** `{expressionNode.GetType().Name}`
@@ -91,6 +103,41 @@ internal class HoverHandler(WorkspaceManager workspaceManager) : HoverHandlerBas
}; };
} }
private static string CreateStructInitializerMessage(StructInitializerNode structInitializerNode, int line, int character)
{
var hoveredInitializerName = structInitializerNode
.Initializers
.Select(x => x.Key)
.FirstOrDefault(x => x.ContainsPosition(line, character));
var structType = (NubStructType)structInitializerNode.Type;
if (hoveredInitializerName != null)
{
var field = structType.Fields.FirstOrDefault(x => x.Name == hoveredInitializerName.Value);
if (field != null)
{
return $"""
**Field** in `{structType}`
```nub
{hoveredInitializerName.Value}: {field.Type}
```
""";
}
else
{
return $"""
**Field** in `{structType}`
```nub
// Field not found
```
""";
}
}
return $"**Struct initializer** `{structType}`";
}
private static string CreateLiteralMessage(NubType type, string value) private static string CreateLiteralMessage(NubType type, string value)
{ {
return $""" return $"""
@@ -111,13 +158,27 @@ internal class HoverHandler(WorkspaceManager workspaceManager) : HoverHandlerBas
"""; """;
} }
private static string CreateFuncIdentifierMessage(FuncIdentifierNode funcIdentifierNode, CompilationUnit compilationUnit) private static string CreateLocalFuncIdentifierMessage(LocalFuncIdentifierNode funcIdentifierNode, ModuleRepository.Module currentModule)
{ {
var func = compilationUnit.ImportedFunctions.FirstOrDefault(x => x.Module == funcIdentifierNode.Module && x.Name == funcIdentifierNode.Name); if (!currentModule.TryResolveFunc(funcIdentifierNode.NameToken, out var func, out _))
if (func == null)
{ {
return $""" return $"""
**Function** `{funcIdentifierNode.Module}::{funcIdentifierNode.Name}` **Function** `{funcIdentifierNode.NameToken.Value}`
```nub
// Declaration not found
```
""";
}
return CreateFuncPrototypeMessage(func);
}
private static string CreateModuleFuncIdentifierMessage(ModuleFuncIdentifierNode funcIdentifierNode, ModuleRepository repository)
{
if (!repository.TryGet(funcIdentifierNode.ModuleToken, out var module) || !module.TryResolveFunc(funcIdentifierNode.NameToken, out var func, out _))
{
return $"""
**Function** `{funcIdentifierNode.ModuleToken.Value}::{funcIdentifierNode.NameToken.Value}`
```nub ```nub
// Declaration not found // Declaration not found
``` ```
@@ -129,13 +190,13 @@ internal class HoverHandler(WorkspaceManager workspaceManager) : HoverHandlerBas
private static string CreateFuncPrototypeMessage(FuncPrototypeNode funcPrototypeNode) private static string CreateFuncPrototypeMessage(FuncPrototypeNode funcPrototypeNode)
{ {
var parameterText = string.Join(", ", funcPrototypeNode.Parameters.Select(x => $"{x.Name}: {x.Type}")); var parameterText = string.Join(", ", funcPrototypeNode.Parameters.Select(x => $"{x.NameToken.Value}: {x.Type}"));
var externText = funcPrototypeNode.ExternSymbol != null ? $"extern \"{funcPrototypeNode.ExternSymbol}\" " : ""; var externText = funcPrototypeNode.ExternSymbolToken != null ? $"extern \"{funcPrototypeNode.ExternSymbolToken.Value}\" " : "";
return $""" return $"""
**Function** `{funcPrototypeNode.Module}::{funcPrototypeNode.Name}` **Function** `{funcPrototypeNode.NameToken.Value}`
```nub ```nub
{externText}func {funcPrototypeNode.Module}::{funcPrototypeNode.Name}({parameterText}): {funcPrototypeNode.ReturnType} {externText}func {funcPrototypeNode.NameToken.Value}({parameterText}): {funcPrototypeNode.ReturnType}
``` ```
"""; """;
} }

View File

@@ -18,17 +18,7 @@ var server = await LanguageServer.From(options => options
.WithHandler<HoverHandler>() .WithHandler<HoverHandler>()
.WithHandler<CompletionHandler>() .WithHandler<CompletionHandler>()
.WithHandler<DefinitionHandler>() .WithHandler<DefinitionHandler>()
.OnInitialize((server, request, ct) => .WithHandler<SetRootPathCommandHandler>()
{
var workspaceManager = server.GetRequiredService<WorkspaceManager>();
if (request.RootPath != null)
{
workspaceManager.Init(request.RootPath);
}
return Task.CompletedTask;
})
); );
await server.WaitForExit; await server.WaitForExit;

View File

@@ -0,0 +1,31 @@
using MediatR;
using OmniSharp.Extensions.LanguageServer.Protocol.Client.Capabilities;
using OmniSharp.Extensions.LanguageServer.Protocol.Models;
using OmniSharp.Extensions.LanguageServer.Protocol.Workspace;
namespace NubLang.LSP;
public class SetRootPathCommandHandler(WorkspaceManager workspaceManager) : ExecuteCommandHandlerBase
{
protected override ExecuteCommandRegistrationOptions CreateRegistrationOptions(ExecuteCommandCapability capability, ClientCapabilities clientCapabilities)
{
return new ExecuteCommandRegistrationOptions
{
Commands = new Container<string>("nub.setRootPath")
};
}
public override Task<Unit> Handle(ExecuteCommandParams request, CancellationToken cancellationToken)
{
if (request is { Command: "nub.setRootPath", Arguments.Count: > 0 })
{
var newRoot = request.Arguments[0].ToString();
if (!string.IsNullOrEmpty(newRoot))
{
workspaceManager.SetRootPath(newRoot);
}
}
return Unit.Task;
}
}

View File

@@ -15,25 +15,25 @@ internal class TextDocumentSyncHandler(WorkspaceManager workspaceManager) : Text
public override Task<Unit> Handle(DidOpenTextDocumentParams request, CancellationToken cancellationToken) public override Task<Unit> Handle(DidOpenTextDocumentParams request, CancellationToken cancellationToken)
{ {
workspaceManager.UpdateFile(request.TextDocument.Uri.GetFileSystemPath()); workspaceManager.Update();
return Unit.Task; return Unit.Task;
} }
public override Task<Unit> Handle(DidChangeTextDocumentParams request, CancellationToken cancellationToken) public override Task<Unit> Handle(DidChangeTextDocumentParams request, CancellationToken cancellationToken)
{ {
workspaceManager.UpdateFile(request.TextDocument.Uri.GetFileSystemPath()); workspaceManager.Update();
return Unit.Task; return Unit.Task;
} }
public override Task<Unit> Handle(DidSaveTextDocumentParams request, CancellationToken cancellationToken) public override Task<Unit> Handle(DidSaveTextDocumentParams request, CancellationToken cancellationToken)
{ {
workspaceManager.UpdateFile(request.TextDocument.Uri.GetFileSystemPath()); workspaceManager.Update();
return Unit.Task; return Unit.Task;
} }
public override Task<Unit> Handle(DidCloseTextDocumentParams request, CancellationToken cancellationToken) public override Task<Unit> Handle(DidCloseTextDocumentParams request, CancellationToken cancellationToken)
{ {
workspaceManager.UpdateFile(request.TextDocument.Uri.GetFileSystemPath()); workspaceManager.Update();
return Unit.Task; return Unit.Task;
} }

View File

@@ -1,74 +1,68 @@
using NubLang.Ast; using NubLang.Ast;
using NubLang.Diagnostics;
using NubLang.Modules;
using NubLang.Syntax; using NubLang.Syntax;
using OmniSharp.Extensions.LanguageServer.Protocol;
namespace NubLang.LSP; namespace NubLang.LSP;
public class WorkspaceManager(DiagnosticsPublisher diagnosticsPublisher) public class WorkspaceManager(DiagnosticsPublisher diagnosticsPublisher)
{ {
private readonly Dictionary<string, SyntaxTree> _syntaxTrees = new(); private record Unit(SyntaxTree SyntaxTree, DateTimeOffset FileTimestamp, List<Diagnostic> Diagnostics);
private readonly Dictionary<string, CompilationUnit> _compilationUnits = new();
public void Init(string rootPath) private readonly Tokenizer _tokenizer = new();
private readonly Parser _parser = new();
private readonly TypeChecker _typeChecker = new();
private string? _rootPath;
private readonly Dictionary<string, Unit> _units = [];
private readonly Dictionary<string, List<TopLevelNode>> _possiblyOutdatedTopLevelNodes = [];
private ModuleRepository _repository = new([]);
public void SetRootPath(string rootPath)
{ {
var files = Directory.GetFiles(rootPath, "*.nub", SearchOption.AllDirectories); _rootPath = rootPath;
foreach (var path in files) Update();
}
public void Update()
{
if (_rootPath == null) return;
var files = Directory.GetFiles(_rootPath, "*.nub", SearchOption.AllDirectories);
foreach (var file in files)
{ {
var text = File.ReadAllText(path); var lastUpdated = File.GetLastWriteTimeUtc(file);
var tokenizer = new Tokenizer(path, text); var unit = _units.GetValueOrDefault(file);
if (unit == null || lastUpdated > unit.FileTimestamp)
tokenizer.Tokenize(); {
diagnosticsPublisher.Publish(path, tokenizer.Diagnostics); _units[file] = Update(file, lastUpdated);
}
var parser = new Parser();
var parseResult = parser.Parse(tokenizer.Tokens);
diagnosticsPublisher.Publish(path, parser.Diagnostics);
_syntaxTrees[path] = parseResult;
} }
foreach (var (fsPath, syntaxTree) in _syntaxTrees) _repository = ModuleRepository.Create(_units.Select(x => x.Value.SyntaxTree).ToList());
{
var modules = Module.Collect(_syntaxTrees.Select(x => x.Value).ToList());
var typeChecker = new TypeChecker(syntaxTree, modules); foreach (var (file, unit) in _units)
var result = typeChecker.Check(); {
diagnosticsPublisher.Publish(fsPath, typeChecker.Diagnostics); var topLevelNodes = _typeChecker.Check(unit.SyntaxTree, _repository);
_compilationUnits[fsPath] = result; _possiblyOutdatedTopLevelNodes[file] = topLevelNodes;
diagnosticsPublisher.Publish(file, [..unit.Diagnostics, .._typeChecker.Diagnostics]);
} }
} }
public void UpdateFile(DocumentUri path) private Unit Update(string file, DateTimeOffset lastUpdated)
{ {
var fsPath = path.GetFileSystemPath(); var text = File.ReadAllText(file);
var tokens = _tokenizer.Tokenize(file, text);
var syntaxTree = _parser.Parse(tokens);
var text = File.ReadAllText(fsPath); return new Unit(syntaxTree, lastUpdated, [.._tokenizer.Diagnostics, .._parser.Diagnostics]);
var tokenizer = new Tokenizer(fsPath, text);
tokenizer.Tokenize();
diagnosticsPublisher.Publish(path, tokenizer.Diagnostics);
var parser = new Parser();
var syntaxTree = parser.Parse(tokenizer.Tokens);
diagnosticsPublisher.Publish(path, parser.Diagnostics);
_syntaxTrees[fsPath] = syntaxTree;
var modules = Module.Collect(_syntaxTrees.Select(x => x.Value).ToList());
var typeChecker = new TypeChecker(syntaxTree, modules);
var result = typeChecker.Check();
diagnosticsPublisher.Publish(fsPath, typeChecker.Diagnostics);
_compilationUnits[fsPath] = result;
} }
public void RemoveFile(DocumentUri path) public List<TopLevelNode> GetTopLevelNodes(string path)
{ {
var fsPath = path.GetFileSystemPath(); return _possiblyOutdatedTopLevelNodes.GetValueOrDefault(path, []);
_syntaxTrees.Remove(fsPath);
_compilationUnits.Remove(fsPath);
} }
public CompilationUnit? GetCompilationUnit(DocumentUri path) public ModuleRepository GetModuleRepository()
{ {
return _compilationUnits.GetValueOrDefault(path.GetFileSystemPath()); return _repository;
} }
} }

View File

@@ -1,15 +0,0 @@
namespace NubLang.Ast;
public sealed class CompilationUnit
{
public CompilationUnit(List<FuncNode> functions, List<NubStructType> importedStructTypes, List<FuncPrototypeNode> importedFunctions)
{
Functions = functions;
ImportedStructTypes = importedStructTypes;
ImportedFunctions = importedFunctions;
}
public List<FuncNode> Functions { get; }
public List<NubStructType> ImportedStructTypes { get; }
public List<FuncPrototypeNode> ImportedFunctions { get; }
}

View File

@@ -1,4 +1,5 @@
using NubLang.Syntax; using NubLang.Syntax;
using NubLang.Types;
namespace NubLang.Ast; namespace NubLang.Ast;
@@ -29,17 +30,28 @@ public abstract class Node(List<Token> tokens)
} }
} }
#region Definitions public abstract class TopLevelNode(List<Token> tokens) : Node(tokens);
public abstract class DefinitionNode(List<Token> tokens, string module, string name) : Node(tokens) public class ModuleNode(List<Token> tokens, IdentifierToken nameToken) : TopLevelNode(tokens)
{ {
public string Module { get; } = module; public IdentifierToken NameToken { get; } = nameToken;
public string Name { get; } = name;
public override IEnumerable<Node> Children()
{
return [];
}
} }
public class FuncParameterNode(List<Token> tokens, string name, NubType type) : Node(tokens) #region Definitions
public abstract class DefinitionNode(List<Token> tokens, IdentifierToken nameToken) : TopLevelNode(tokens)
{ {
public string Name { get; } = name; public IdentifierToken NameToken { get; } = nameToken;
}
public class FuncParameterNode(List<Token> tokens, IdentifierToken nameToken, NubType type) : Node(tokens)
{
public IdentifierToken NameToken { get; } = nameToken;
public NubType Type { get; } = type; public NubType Type { get; } = type;
public override IEnumerable<Node> Children() public override IEnumerable<Node> Children()
@@ -48,11 +60,10 @@ public class FuncParameterNode(List<Token> tokens, string name, NubType type) :
} }
} }
public class FuncPrototypeNode(List<Token> tokens, string module, string name, string? externSymbol, List<FuncParameterNode> parameters, NubType returnType) : Node(tokens) public class FuncPrototypeNode(List<Token> tokens, IdentifierToken nameToken, StringLiteralToken? externSymbolToken, List<FuncParameterNode> parameters, NubType returnType) : Node(tokens)
{ {
public string Module { get; } = module; public IdentifierToken NameToken { get; } = nameToken;
public string Name { get; } = name; public StringLiteralToken? ExternSymbolToken { get; } = externSymbolToken;
public string? ExternSymbol { get; } = externSymbol;
public List<FuncParameterNode> Parameters { get; } = parameters; public List<FuncParameterNode> Parameters { get; } = parameters;
public NubType ReturnType { get; } = returnType; public NubType ReturnType { get; } = returnType;
@@ -62,7 +73,7 @@ public class FuncPrototypeNode(List<Token> tokens, string module, string name, s
} }
} }
public class FuncNode(List<Token> tokens, FuncPrototypeNode prototype, BlockNode? body) : DefinitionNode(tokens, prototype.Module, prototype.Name) public class FuncNode(List<Token> tokens, FuncPrototypeNode prototype, BlockNode? body) : DefinitionNode(tokens, prototype.NameToken)
{ {
public FuncPrototypeNode Prototype { get; } = prototype; public FuncPrototypeNode Prototype { get; } = prototype;
public BlockNode? Body { get; } = body; public BlockNode? Body { get; } = body;
@@ -77,14 +88,42 @@ public class FuncNode(List<Token> tokens, FuncPrototypeNode prototype, BlockNode
} }
} }
public class StructFieldNode(List<Token> tokens, IdentifierToken nameToken, NubType type, ExpressionNode? value) : Node(tokens)
{
public IdentifierToken NameToken { get; } = nameToken;
public NubType Type { get; } = type;
public ExpressionNode? Value { get; } = value;
public override IEnumerable<Node> Children()
{
if (Value != null)
{
yield return Value;
}
}
}
public class StructNode(List<Token> tokens, IdentifierToken name, NubStructType structType, bool packed, List<StructFieldNode> fields) : DefinitionNode(tokens, name)
{
public NubStructType StructType { get; } = structType;
public bool Packed { get; } = packed;
public List<StructFieldNode> Fields { get; } = fields;
public override IEnumerable<Node> Children()
{
foreach (var field in Fields)
{
yield return field;
}
}
}
#endregion #endregion
#region Statements #region Statements
public abstract class StatementNode(List<Token> tokens) : Node(tokens); public abstract class StatementNode(List<Token> tokens) : Node(tokens);
public abstract class TerminalStatementNode(List<Token> tokens) : StatementNode(tokens);
public class BlockNode(List<Token> tokens, List<StatementNode> statements) : StatementNode(tokens) public class BlockNode(List<Token> tokens, List<StatementNode> statements) : StatementNode(tokens)
{ {
public List<StatementNode> Statements { get; } = statements; public List<StatementNode> Statements { get; } = statements;
@@ -105,7 +144,7 @@ public class StatementFuncCallNode(List<Token> tokens, FuncCallNode funcCall) :
} }
} }
public class ReturnNode(List<Token> tokens, ExpressionNode? value) : TerminalStatementNode(tokens) public class ReturnNode(List<Token> tokens, ExpressionNode? value) : StatementNode(tokens)
{ {
public ExpressionNode? Value { get; } = value; public ExpressionNode? Value { get; } = value;
@@ -115,9 +154,9 @@ public class ReturnNode(List<Token> tokens, ExpressionNode? value) : TerminalSta
} }
} }
public class AssignmentNode(List<Token> tokens, LValueExpressionNode target, ExpressionNode value) : StatementNode(tokens) public class AssignmentNode(List<Token> tokens, ExpressionNode target, ExpressionNode value) : StatementNode(tokens)
{ {
public LValueExpressionNode Target { get; } = target; public ExpressionNode Target { get; } = target;
public ExpressionNode Value { get; } = value; public ExpressionNode Value { get; } = value;
public override IEnumerable<Node> Children() public override IEnumerable<Node> Children()
@@ -144,9 +183,9 @@ public class IfNode(List<Token> tokens, ExpressionNode condition, BlockNode body
} }
} }
public class VariableDeclarationNode(List<Token> tokens, string name, ExpressionNode? assignment, NubType type) : StatementNode(tokens) public class VariableDeclarationNode(List<Token> tokens, IdentifierToken nameToken, ExpressionNode? assignment, NubType type) : StatementNode(tokens)
{ {
public string Name { get; } = name; public IdentifierToken NameToken { get; } = nameToken;
public ExpressionNode? Assignment { get; } = assignment; public ExpressionNode? Assignment { get; } = assignment;
public NubType Type { get; } = type; public NubType Type { get; } = type;
@@ -156,7 +195,7 @@ public class VariableDeclarationNode(List<Token> tokens, string name, Expression
} }
} }
public class ContinueNode(List<Token> tokens) : TerminalStatementNode(tokens) public class ContinueNode(List<Token> tokens) : StatementNode(tokens)
{ {
public override IEnumerable<Node> Children() public override IEnumerable<Node> Children()
{ {
@@ -164,7 +203,7 @@ public class ContinueNode(List<Token> tokens) : TerminalStatementNode(tokens)
} }
} }
public class BreakNode(List<Token> tokens) : TerminalStatementNode(tokens) public class BreakNode(List<Token> tokens) : StatementNode(tokens)
{ {
public override IEnumerable<Node> Children() public override IEnumerable<Node> Children()
{ {
@@ -184,10 +223,10 @@ public class WhileNode(List<Token> tokens, ExpressionNode condition, BlockNode b
} }
} }
public class ForSliceNode(List<Token> tokens, string elementName, string? indexName, ExpressionNode target, BlockNode body) : StatementNode(tokens) public class ForSliceNode(List<Token> tokens, IdentifierToken elementNameToken, IdentifierToken? indexNameToken, ExpressionNode target, BlockNode body) : StatementNode(tokens)
{ {
public string ElementName { get; } = elementName; public IdentifierToken ElementNameToken { get; } = elementNameToken;
public string? IndexName { get; } = indexName; public IdentifierToken? IndexNameToken { get; } = indexNameToken;
public ExpressionNode Target { get; } = target; public ExpressionNode Target { get; } = target;
public BlockNode Body { get; } = body; public BlockNode Body { get; } = body;
@@ -198,10 +237,10 @@ public class ForSliceNode(List<Token> tokens, string elementName, string? indexN
} }
} }
public class ForConstArrayNode(List<Token> tokens, string elementName, string? indexName, ExpressionNode target, BlockNode body) : StatementNode(tokens) public class ForConstArrayNode(List<Token> tokens, IdentifierToken elementNameToken, IdentifierToken? indexNameToken, ExpressionNode target, BlockNode body) : StatementNode(tokens)
{ {
public string ElementName { get; } = elementName; public IdentifierToken ElementNameToken { get; } = elementNameToken;
public string? IndexName { get; } = indexName; public IdentifierToken? IndexNameToken { get; } = indexNameToken;
public ExpressionNode Target { get; } = target; public ExpressionNode Target { get; } = target;
public BlockNode Body { get; } = body; public BlockNode Body { get; } = body;
@@ -259,13 +298,11 @@ public abstract class ExpressionNode(List<Token> tokens, NubType type) : Node(to
public NubType Type { get; } = type; public NubType Type { get; } = type;
} }
public abstract class LValueExpressionNode(List<Token> tokens, NubType type) : ExpressionNode(tokens, type); public abstract class LValue(List<Token> tokens, NubType type) : ExpressionNode(tokens, type);
public abstract class RValueExpressionNode(List<Token> tokens, NubType type) : ExpressionNode(tokens, type); public abstract class RValue(List<Token> tokens, NubType type) : ExpressionNode(tokens, type);
public abstract class IntermediateExpression(List<Token> tokens) : ExpressionNode(tokens, new NubVoidType()); public class StringLiteralNode(List<Token> tokens, string value) : RValue(tokens, new NubStringType())
public class StringLiteralNode(List<Token> tokens, string value) : RValueExpressionNode(tokens, new NubStringType())
{ {
public string Value { get; } = value; public string Value { get; } = value;
@@ -275,7 +312,7 @@ public class StringLiteralNode(List<Token> tokens, string value) : RValueExpress
} }
} }
public class CStringLiteralNode(List<Token> tokens, string value) : RValueExpressionNode(tokens, new NubPointerType(new NubIntType(true, 8))) public class CStringLiteralNode(List<Token> tokens, string value) : RValue(tokens, new NubPointerType(new NubIntType(true, 8)))
{ {
public string Value { get; } = value; public string Value { get; } = value;
@@ -285,7 +322,7 @@ public class CStringLiteralNode(List<Token> tokens, string value) : RValueExpres
} }
} }
public class I8LiteralNode(List<Token> tokens, sbyte value) : RValueExpressionNode(tokens, new NubIntType(true, 8)) public class I8LiteralNode(List<Token> tokens, sbyte value) : RValue(tokens, new NubIntType(true, 8))
{ {
public sbyte Value { get; } = value; public sbyte Value { get; } = value;
@@ -295,7 +332,7 @@ public class I8LiteralNode(List<Token> tokens, sbyte value) : RValueExpressionNo
} }
} }
public class I16LiteralNode(List<Token> tokens, short value) : RValueExpressionNode(tokens, new NubIntType(true, 16)) public class I16LiteralNode(List<Token> tokens, short value) : RValue(tokens, new NubIntType(true, 16))
{ {
public short Value { get; } = value; public short Value { get; } = value;
@@ -305,7 +342,7 @@ public class I16LiteralNode(List<Token> tokens, short value) : RValueExpressionN
} }
} }
public class I32LiteralNode(List<Token> tokens, int value) : RValueExpressionNode(tokens, new NubIntType(true, 32)) public class I32LiteralNode(List<Token> tokens, int value) : RValue(tokens, new NubIntType(true, 32))
{ {
public int Value { get; } = value; public int Value { get; } = value;
@@ -315,7 +352,7 @@ public class I32LiteralNode(List<Token> tokens, int value) : RValueExpressionNod
} }
} }
public class I64LiteralNode(List<Token> tokens, long value) : RValueExpressionNode(tokens, new NubIntType(true, 64)) public class I64LiteralNode(List<Token> tokens, long value) : RValue(tokens, new NubIntType(true, 64))
{ {
public long Value { get; } = value; public long Value { get; } = value;
@@ -325,7 +362,7 @@ public class I64LiteralNode(List<Token> tokens, long value) : RValueExpressionNo
} }
} }
public class U8LiteralNode(List<Token> tokens, byte value) : RValueExpressionNode(tokens, new NubIntType(false, 8)) public class U8LiteralNode(List<Token> tokens, byte value) : RValue(tokens, new NubIntType(false, 8))
{ {
public byte Value { get; } = value; public byte Value { get; } = value;
@@ -335,7 +372,7 @@ public class U8LiteralNode(List<Token> tokens, byte value) : RValueExpressionNod
} }
} }
public class U16LiteralNode(List<Token> tokens, ushort value) : RValueExpressionNode(tokens, new NubIntType(false, 16)) public class U16LiteralNode(List<Token> tokens, ushort value) : RValue(tokens, new NubIntType(false, 16))
{ {
public ushort Value { get; } = value; public ushort Value { get; } = value;
@@ -345,7 +382,7 @@ public class U16LiteralNode(List<Token> tokens, ushort value) : RValueExpression
} }
} }
public class U32LiteralNode(List<Token> tokens, uint value) : RValueExpressionNode(tokens, new NubIntType(false, 32)) public class U32LiteralNode(List<Token> tokens, uint value) : RValue(tokens, new NubIntType(false, 32))
{ {
public uint Value { get; } = value; public uint Value { get; } = value;
@@ -355,7 +392,7 @@ public class U32LiteralNode(List<Token> tokens, uint value) : RValueExpressionNo
} }
} }
public class U64LiteralNode(List<Token> tokens, ulong value) : RValueExpressionNode(tokens, new NubIntType(false, 64)) public class U64LiteralNode(List<Token> tokens, ulong value) : RValue(tokens, new NubIntType(false, 64))
{ {
public ulong Value { get; } = value; public ulong Value { get; } = value;
@@ -365,7 +402,7 @@ public class U64LiteralNode(List<Token> tokens, ulong value) : RValueExpressionN
} }
} }
public class Float32LiteralNode(List<Token> tokens, float value) : RValueExpressionNode(tokens, new NubFloatType(32)) public class Float32LiteralNode(List<Token> tokens, float value) : RValue(tokens, new NubFloatType(32))
{ {
public float Value { get; } = value; public float Value { get; } = value;
@@ -375,7 +412,7 @@ public class Float32LiteralNode(List<Token> tokens, float value) : RValueExpress
} }
} }
public class Float64LiteralNode(List<Token> tokens, double value) : RValueExpressionNode(tokens, new NubFloatType(64)) public class Float64LiteralNode(List<Token> tokens, double value) : RValue(tokens, new NubFloatType(64))
{ {
public double Value { get; } = value; public double Value { get; } = value;
@@ -385,7 +422,7 @@ public class Float64LiteralNode(List<Token> tokens, double value) : RValueExpres
} }
} }
public class BoolLiteralNode(List<Token> tokens, NubType type, bool value) : RValueExpressionNode(tokens, type) public class BoolLiteralNode(List<Token> tokens, bool value) : RValue(tokens, new NubBoolType())
{ {
public bool Value { get; } = value; public bool Value { get; } = value;
@@ -395,7 +432,7 @@ public class BoolLiteralNode(List<Token> tokens, NubType type, bool value) : RVa
} }
} }
public class BinaryExpressionNode(List<Token> tokens, NubType type, ExpressionNode left, BinaryOperator @operator, ExpressionNode right) : RValueExpressionNode(tokens, type) public class BinaryExpressionNode(List<Token> tokens, NubType type, ExpressionNode left, BinaryOperator @operator, ExpressionNode right) : RValue(tokens, type)
{ {
public ExpressionNode Left { get; } = left; public ExpressionNode Left { get; } = left;
public BinaryOperator Operator { get; } = @operator; public BinaryOperator Operator { get; } = @operator;
@@ -408,7 +445,7 @@ public class BinaryExpressionNode(List<Token> tokens, NubType type, ExpressionNo
} }
} }
public class UnaryExpressionNode(List<Token> tokens, NubType type, UnaryOperator @operator, ExpressionNode operand) : RValueExpressionNode(tokens, type) public class UnaryExpressionNode(List<Token> tokens, NubType type, UnaryOperator @operator, ExpressionNode operand) : RValue(tokens, type)
{ {
public UnaryOperator Operator { get; } = @operator; public UnaryOperator Operator { get; } = @operator;
public ExpressionNode Operand { get; } = operand; public ExpressionNode Operand { get; } = operand;
@@ -419,7 +456,7 @@ public class UnaryExpressionNode(List<Token> tokens, NubType type, UnaryOperator
} }
} }
public class FuncCallNode(List<Token> tokens, NubType type, ExpressionNode expression, List<ExpressionNode> parameters) : RValueExpressionNode(tokens, type) public class FuncCallNode(List<Token> tokens, NubType type, ExpressionNode expression, List<ExpressionNode> parameters) : RValue(tokens, type)
{ {
public ExpressionNode Expression { get; } = expression; public ExpressionNode Expression { get; } = expression;
public List<ExpressionNode> Parameters { get; } = parameters; public List<ExpressionNode> Parameters { get; } = parameters;
@@ -434,9 +471,9 @@ public class FuncCallNode(List<Token> tokens, NubType type, ExpressionNode expre
} }
} }
public class VariableIdentifierNode(List<Token> tokens, NubType type, string name) : LValueExpressionNode(tokens, type) public class VariableIdentifierNode(List<Token> tokens, NubType type, IdentifierToken nameToken) : LValue(tokens, type)
{ {
public string Name { get; } = name; public IdentifierToken NameToken { get; } = nameToken;
public override IEnumerable<Node> Children() public override IEnumerable<Node> Children()
{ {
@@ -444,11 +481,10 @@ public class VariableIdentifierNode(List<Token> tokens, NubType type, string nam
} }
} }
public class FuncIdentifierNode(List<Token> tokens, NubType type, string module, string name, string? externSymbol) : RValueExpressionNode(tokens, type) public class LocalFuncIdentifierNode(List<Token> tokens, NubType type, IdentifierToken nameToken, StringLiteralToken? externSymbolToken) : RValue(tokens, type)
{ {
public string Module { get; } = module; public IdentifierToken NameToken { get; } = nameToken;
public string Name { get; } = name; public StringLiteralToken? ExternSymbolToken { get; } = externSymbolToken;
public string? ExternSymbol { get; } = externSymbol;
public override IEnumerable<Node> Children() public override IEnumerable<Node> Children()
{ {
@@ -456,27 +492,19 @@ public class FuncIdentifierNode(List<Token> tokens, NubType type, string module,
} }
} }
public class ArrayInitializerNode(List<Token> tokens, NubType type, List<ExpressionNode> values) : RValueExpressionNode(tokens, type) public class ModuleFuncIdentifierNode(List<Token> tokens, NubType type, IdentifierToken moduleToken, IdentifierToken nameToken, StringLiteralToken? externSymbolToken) : RValue(tokens, type)
{ {
public List<ExpressionNode> Values { get; } = values; public IdentifierToken ModuleToken { get; } = moduleToken;
public IdentifierToken NameToken { get; } = nameToken;
public StringLiteralToken? ExternSymbolToken { get; } = externSymbolToken;
public override IEnumerable<Node> Children() public override IEnumerable<Node> Children()
{ {
return Values; return [];
} }
} }
public class ConstArrayInitializerNode(List<Token> tokens, NubType type, List<ExpressionNode> values) : RValueExpressionNode(tokens, type) public class ArrayIndexAccessNode(List<Token> tokens, NubType type, ExpressionNode target, ExpressionNode index) : LValue(tokens, type)
{
public List<ExpressionNode> Values { get; } = values;
public override IEnumerable<Node> Children()
{
return Values;
}
}
public class ArrayIndexAccessNode(List<Token> tokens, NubType type, ExpressionNode target, ExpressionNode index) : LValueExpressionNode(tokens, type)
{ {
public ExpressionNode Target { get; } = target; public ExpressionNode Target { get; } = target;
public ExpressionNode Index { get; } = index; public ExpressionNode Index { get; } = index;
@@ -488,7 +516,7 @@ public class ArrayIndexAccessNode(List<Token> tokens, NubType type, ExpressionNo
} }
} }
public class ConstArrayIndexAccessNode(List<Token> tokens, NubType type, ExpressionNode target, ExpressionNode index) : LValueExpressionNode(tokens, type) public class ConstArrayIndexAccessNode(List<Token> tokens, NubType type, ExpressionNode target, ExpressionNode index) : LValue(tokens, type)
{ {
public ExpressionNode Target { get; } = target; public ExpressionNode Target { get; } = target;
public ExpressionNode Index { get; } = index; public ExpressionNode Index { get; } = index;
@@ -500,7 +528,7 @@ public class ConstArrayIndexAccessNode(List<Token> tokens, NubType type, Express
} }
} }
public class SliceIndexAccessNode(List<Token> tokens, NubType type, ExpressionNode target, ExpressionNode index) : LValueExpressionNode(tokens, type) public class SliceIndexAccessNode(List<Token> tokens, NubType type, ExpressionNode target, ExpressionNode index) : LValue(tokens, type)
{ {
public ExpressionNode Target { get; } = target; public ExpressionNode Target { get; } = target;
public ExpressionNode Index { get; } = index; public ExpressionNode Index { get; } = index;
@@ -512,20 +540,9 @@ public class SliceIndexAccessNode(List<Token> tokens, NubType type, ExpressionNo
} }
} }
public class AddressOfNode(List<Token> tokens, NubType type, LValueExpressionNode lValue) : RValueExpressionNode(tokens, type) public class AddressOfNode(List<Token> tokens, NubType type, ExpressionNode target) : RValue(tokens, type)
{
public LValueExpressionNode LValue { get; } = lValue;
public override IEnumerable<Node> Children()
{
yield return LValue;
}
}
public class StructFieldAccessNode(List<Token> tokens, NubType type, ExpressionNode target, string field) : LValueExpressionNode(tokens, type)
{ {
public ExpressionNode Target { get; } = target; public ExpressionNode Target { get; } = target;
public string Field { get; } = field;
public override IEnumerable<Node> Children() public override IEnumerable<Node> Children()
{ {
@@ -533,9 +550,68 @@ public class StructFieldAccessNode(List<Token> tokens, NubType type, ExpressionN
} }
} }
public class StructInitializerNode(List<Token> tokens, NubType type, Dictionary<string, ExpressionNode> initializers) : RValueExpressionNode(tokens, type) public class StructFieldAccessNode(List<Token> tokens, NubType type, ExpressionNode target, IdentifierToken fieldToken) : LValue(tokens, type)
{ {
public Dictionary<string, ExpressionNode> Initializers { get; } = initializers; public ExpressionNode Target { get; } = target;
public IdentifierToken FieldToken { get; } = fieldToken;
public override IEnumerable<Node> Children()
{
yield return Target;
}
}
public class DereferenceNode(List<Token> tokens, NubType type, ExpressionNode target) : LValue(tokens, type)
{
public ExpressionNode Target { get; } = target;
public override IEnumerable<Node> Children()
{
yield return Target;
}
}
public class SizeNode(List<Token> tokens, NubType targetType) : RValue(tokens, new NubIntType(false, 64))
{
public NubType TargetType { get; } = targetType;
public override IEnumerable<Node> Children()
{
return [];
}
}
public class CastNode(List<Token> tokens, NubType type, ExpressionNode value, CastNode.Conversion conversionType) : RValue(tokens, type)
{
public enum Conversion
{
IntToInt,
FloatToFloat,
IntToFloat,
FloatToInt,
PointerToPointer,
PointerToUInt64,
UInt64ToPointer,
ConstArrayToArray,
ConstArrayToSlice,
StringToCString
}
public ExpressionNode Value { get; } = value;
public Conversion ConversionType { get; } = conversionType;
public override IEnumerable<Node> Children()
{
yield return Value;
}
}
public class StructInitializerNode(List<Token> tokens, NubType type, Dictionary<IdentifierToken, ExpressionNode> initializers) : LValue(tokens, type)
{
public Dictionary<IdentifierToken, ExpressionNode> Initializers { get; } = initializers;
public override IEnumerable<Node> Children() public override IEnumerable<Node> Children()
{ {
@@ -546,44 +622,13 @@ public class StructInitializerNode(List<Token> tokens, NubType type, Dictionary<
} }
} }
public class DereferenceNode(List<Token> tokens, NubType type, ExpressionNode target) : LValueExpressionNode(tokens, type) public class ConstArrayInitializerNode(List<Token> tokens, NubType type, List<ExpressionNode> values) : LValue(tokens, type)
{ {
public ExpressionNode Target { get; } = target; public List<ExpressionNode> Values { get; } = values;
public override IEnumerable<Node> Children() public override IEnumerable<Node> Children()
{ {
yield return Target; return Values;
}
}
public class SizeNode(List<Token> tokens, NubType TargetType) : RValueExpressionNode(tokens, new NubIntType(false, 64))
{
public NubType TargetType { get; } = TargetType;
public override IEnumerable<Node> Children()
{
return [];
}
}
public class CastNode(List<Token> tokens, NubType type, ExpressionNode value) : RValueExpressionNode(tokens, type)
{
public ExpressionNode Value { get; } = value;
public override IEnumerable<Node> Children()
{
yield return Value;
}
}
public class EnumReferenceIntermediateNode(List<Token> tokens, string module, string name) : IntermediateExpression(tokens)
{
public string Module { get; } = module;
public string Name { get; } = name;
public override IEnumerable<Node> Children()
{
return [];
} }
} }

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,11 @@
namespace NubLang.Diagnostics;
public class CompileException : Exception
{
public Diagnostic Diagnostic { get; }
public CompileException(Diagnostic diagnostic) : base(diagnostic.Message)
{
Diagnostic = diagnostic;
}
}

View File

@@ -11,6 +11,7 @@ public class Diagnostic
private readonly string _message; private readonly string _message;
private SourceSpan? _span; private SourceSpan? _span;
private string? _help; private string? _help;
private List<Token>? _tokens;
public DiagnosticBuilder(DiagnosticSeverity severity, string message) public DiagnosticBuilder(DiagnosticSeverity severity, string message)
{ {
@@ -18,18 +19,32 @@ public class Diagnostic
_message = message; _message = message;
} }
public DiagnosticBuilder At(SyntaxNode? node) public DiagnosticBuilder At(SyntaxNode? node, List<Token>? tokens = null)
{ {
if (tokens != null)
{
_tokens = tokens;
}
if (node != null) if (node != null)
{ {
_span = SourceSpan.Merge(node.Tokens.Select(x => x.Span)); var first = node.Tokens.FirstOrDefault();
if (first != null)
{
_span = SourceSpan.Merge(node.Tokens.Select(x => x.Span));
}
} }
return this; return this;
} }
public DiagnosticBuilder At(Token? token) public DiagnosticBuilder At(Token? token, List<Token>? tokens = null)
{ {
if (tokens != null)
{
_tokens = tokens;
}
if (token != null) if (token != null)
{ {
At(token.Span); At(token.Span);
@@ -48,11 +63,11 @@ public class Diagnostic
return this; return this;
} }
public DiagnosticBuilder At(string filePath, int line, int column) // public DiagnosticBuilder At(string filePath, int line, int column)
{ // {
_span = new SourceSpan(filePath, new SourceLocation(line, column), new SourceLocation(line, column)); // _span = new SourceSpan(filePath, new SourceLocation(line, column), new SourceLocation(line, column));
return this; // return this;
} // }
public DiagnosticBuilder WithHelp(string help) public DiagnosticBuilder WithHelp(string help)
{ {
@@ -60,20 +75,23 @@ public class Diagnostic
return this; return this;
} }
public Diagnostic Build() => new(_severity, _message, _help, _span); public Diagnostic Build() => new(_severity, _message, _help, _span, _tokens);
} }
public static DiagnosticBuilder Error(string message) => new(DiagnosticSeverity.Error, message); public static DiagnosticBuilder Error(string message) => new(DiagnosticSeverity.Error, message);
public static DiagnosticBuilder Warning(string message) => new(DiagnosticSeverity.Warning, message); public static DiagnosticBuilder Warning(string message) => new(DiagnosticSeverity.Warning, message);
public static DiagnosticBuilder Info(string message) => new(DiagnosticSeverity.Info, message); public static DiagnosticBuilder Info(string message) => new(DiagnosticSeverity.Info, message);
private readonly List<Token>? _tokens;
public DiagnosticSeverity Severity { get; } public DiagnosticSeverity Severity { get; }
public string Message { get; } public string Message { get; }
public string? Help { get; } public string? Help { get; }
public SourceSpan? Span { get; } public SourceSpan? Span { get; }
private Diagnostic(DiagnosticSeverity severity, string message, string? help, SourceSpan? span) private Diagnostic(DiagnosticSeverity severity, string message, string? help, SourceSpan? span, List<Token>? tokens)
{ {
_tokens = tokens;
Severity = severity; Severity = severity;
Message = message; Message = message;
Help = help; Help = help;
@@ -103,15 +121,12 @@ public class Diagnostic
if (Span.HasValue) if (Span.HasValue)
{ {
sb.AppendLine(); sb.AppendLine();
var text = File.ReadAllText(Span.Value.FilePath); var text = Span.Value.Source;
var tokenizer = new Tokenizer(Span.Value.FilePath, text);
tokenizer.Tokenize();
var lines = text.Split('\n'); var lines = text.Split('\n');
var startLine = Span.Value.Start.Line; var startLine = Span.Value.StartLine;
var endLine = Span.Value.End.Line; var endLine = Span.Value.EndLine;
const int CONTEXT_LINES = 3; const int CONTEXT_LINES = 3;
@@ -144,24 +159,31 @@ public class Diagnostic
sb.Append("│ "); sb.Append("│ ");
sb.Append(i.ToString().PadRight(numberPadding)); sb.Append(i.ToString().PadRight(numberPadding));
sb.Append(" │ "); sb.Append(" │ ");
sb.Append(ApplySyntaxHighlighting(line.PadRight(codePadding), i, tokenizer.Tokens)); if (_tokens != null)
// sb.Append(line.PadRight(codePadding)); {
sb.Append(ApplySyntaxHighlighting(line.PadRight(codePadding), i, _tokens));
}
else
{
sb.Append(line.PadRight(codePadding));
}
sb.Append(" │"); sb.Append(" │");
sb.AppendLine(); sb.AppendLine();
if (i >= startLine && i <= endLine) if (i >= startLine && i <= endLine)
{ {
var markerStartColumn = 1; var markerStartColumn = 1;
var markerEndColumn = line.Length; var markerEndColumn = line.Length + 1;
if (i == startLine) if (i == startLine)
{ {
markerStartColumn = Span.Value.Start.Column; markerStartColumn = Span.Value.StartColumn;
} }
if (i == endLine) if (i == endLine)
{ {
markerEndColumn = Span.Value.End.Column; markerEndColumn = Span.Value.EndColumn;
} }
var markerLength = markerEndColumn - markerStartColumn; var markerLength = markerEndColumn - markerStartColumn;
@@ -206,8 +228,8 @@ public class Diagnostic
{ {
var sb = new StringBuilder(); var sb = new StringBuilder();
var lineTokens = tokens var lineTokens = tokens
.Where(t => t.Span.Start.Line == lineNumber) .Where(t => t.Span.StartLine == lineNumber)
.OrderBy(t => t.Span.Start.Column) .OrderBy(t => t.Span.StartColumn)
.ToList(); .ToList();
if (lineTokens.Count == 0) if (lineTokens.Count == 0)
@@ -219,8 +241,10 @@ public class Diagnostic
foreach (var token in lineTokens) foreach (var token in lineTokens)
{ {
var tokenStart = token.Span.Start.Column; if (token is WhitespaceToken) continue;
var tokenEnd = token.Span.End.Column;
var tokenStart = token.Span.StartColumn;
var tokenEnd = token.Span.EndColumn;
if (tokenStart > currentColumn && currentColumn - 1 < line.Length) if (tokenStart > currentColumn && currentColumn - 1 < line.Length)
{ {
@@ -262,6 +286,10 @@ public class Diagnostic
{ {
switch (token) switch (token)
{ {
case CommentToken:
{
return ConsoleColors.Colorize(tokenText, ConsoleColors.Green);
}
case IdentifierToken: case IdentifierToken:
{ {
return ConsoleColors.Colorize(tokenText, ConsoleColors.BrightWhite); return ConsoleColors.Colorize(tokenText, ConsoleColors.BrightWhite);

View File

@@ -1,112 +1,56 @@
namespace NubLang.Diagnostics; namespace NubLang.Diagnostics;
public readonly struct SourceSpan : IEquatable<SourceSpan>, IComparable<SourceSpan> public readonly struct SourceSpan
{ {
private readonly int _startIndex;
private readonly int _endIndex;
public static SourceSpan Merge(params IEnumerable<SourceSpan> spans) public static SourceSpan Merge(params IEnumerable<SourceSpan> spans)
{ {
var spanArray = spans as SourceSpan[] ?? spans.ToArray(); var spanArray = spans as SourceSpan[] ?? spans.ToArray();
if (spanArray.Length == 0) if (spanArray.Length == 0)
{ {
return new SourceSpan(string.Empty, new SourceLocation(0, 0), new SourceLocation(0, 0)); return new SourceSpan(string.Empty, string.Empty, 0, 0, 0, 0, 0, 0);
} }
var minStart = spanArray.Min(s => s.Start); var first = spanArray.MinBy(x => x._startIndex);
var maxEnd = spanArray.Max(s => s.End); var last = spanArray.MaxBy(x => x._endIndex);
return new SourceSpan(spanArray[0].FilePath, minStart, maxEnd); return new SourceSpan(first.SourcePath, first.Source, first._startIndex, last._endIndex, first.StartLine, last.EndLine, first.StartColumn, last.EndColumn);
} }
public SourceSpan(string filePath, SourceLocation start, SourceLocation end) public SourceSpan(string sourcePath, string source, int startIndex, int endIndex, int startLine, int startColumn, int endLine, int endColumn)
{ {
if (start > end) _startIndex = startIndex;
{ _endIndex = endIndex;
throw new ArgumentException("Start location cannot be after end location"); SourcePath = sourcePath;
} Source = source;
StartLine = startLine;
FilePath = filePath; StartColumn = startColumn;
Start = start; EndLine = endLine;
End = end; EndColumn = endColumn;
} }
public string FilePath { get; } public int StartLine { get; }
public SourceLocation Start { get; } public int StartColumn { get; }
public SourceLocation End { get; } public int EndLine { get; }
public int EndColumn { get; }
public string SourcePath { get; }
public string Source { get; }
public override string ToString() public override string ToString()
{ {
if (Start == End) if (StartLine == EndLine && StartColumn == EndColumn)
{ {
return $"{FilePath}:{Start}"; return $"{SourcePath}:{StartColumn}:{StartColumn}";
} }
if (Start.Line == End.Line) if (StartLine == EndLine)
{ {
return Start.Column == End.Column ? $"{FilePath}:{Start}" : $"{FilePath}:{Start.Line}:{Start.Column}-{End.Column}"; return $"{SourcePath}:{StartLine}:{StartColumn}-{EndColumn}";
} }
return $"{FilePath}:{Start}-{End}"; return $"{SourcePath}:{StartLine}:{StartColumn}-{EndLine}:{EndColumn}";
}
public bool Equals(SourceSpan other) => Start == other.Start && End == other.End;
public override bool Equals(object? obj) => obj is SourceSpan other && Equals(other);
public override int GetHashCode() => HashCode.Combine(typeof(SourceSpan), Start, End);
public static bool operator ==(SourceSpan left, SourceSpan right) => Equals(left, right);
public static bool operator !=(SourceSpan left, SourceSpan right) => !Equals(left, right);
public static bool operator <(SourceSpan left, SourceSpan right) => left.CompareTo(right) < 0;
public static bool operator <=(SourceSpan left, SourceSpan right) => left.CompareTo(right) <= 0;
public static bool operator >(SourceSpan left, SourceSpan right) => left.CompareTo(right) > 0;
public static bool operator >=(SourceSpan left, SourceSpan right) => left.CompareTo(right) >= 0;
public int CompareTo(SourceSpan other)
{
var startComparison = Start.CompareTo(other.Start);
return startComparison != 0 ? startComparison : End.CompareTo(other.End);
}
}
public readonly struct SourceLocation : IEquatable<SourceLocation>, IComparable<SourceLocation>
{
public SourceLocation(int line, int column)
{
Line = line;
Column = column;
}
public int Line { get; }
public int Column { get; }
public override string ToString()
{
return $"{Line}:{Column}";
}
public override bool Equals(object? obj)
{
return obj is SourceLocation other && Equals(other);
}
public bool Equals(SourceLocation other)
{
return Line == other.Line && Column == other.Column;
}
public override int GetHashCode()
{
return HashCode.Combine(typeof(SourceLocation), Line, Column);
}
public static bool operator ==(SourceLocation left, SourceLocation right) => Equals(left, right);
public static bool operator !=(SourceLocation left, SourceLocation right) => !Equals(left, right);
public static bool operator <(SourceLocation left, SourceLocation right) => left.Line < right.Line || (left.Line == right.Line && left.Column < right.Column);
public static bool operator >(SourceLocation left, SourceLocation right) => left.Line > right.Line || (left.Line == right.Line && left.Column > right.Column);
public static bool operator <=(SourceLocation left, SourceLocation right) => left.Line <= right.Line || (left.Line == right.Line && left.Column <= right.Column);
public static bool operator >=(SourceLocation left, SourceLocation right) => left.Line >= right.Line || (left.Line == right.Line && left.Column >= right.Column);
public int CompareTo(SourceLocation other)
{
var lineComparison = Line.CompareTo(other.Line);
return lineComparison != 0 ? lineComparison : Column.CompareTo(other.Column);
} }
} }

View File

@@ -1,97 +0,0 @@
using NubLang.Ast;
namespace NubLang.Generation;
public static class CType
{
public static string Create(NubType type, string? variableName = null, bool constArraysAsPointers = true)
{
return type switch
{
NubVoidType => "void" + (variableName != null ? $" {variableName}" : ""),
NubBoolType => "bool" + (variableName != null ? $" {variableName}" : ""),
NubIntType intType => CreateIntType(intType, variableName),
NubFloatType floatType => CreateFloatType(floatType, variableName),
NubPointerType ptr => CreatePointerType(ptr, variableName),
NubSliceType => "struct nub_slice" + (variableName != null ? $" {variableName}" : ""),
NubStringType => "struct nub_string" + (variableName != null ? $" {variableName}" : ""),
NubConstArrayType arr => CreateConstArrayType(arr, variableName, constArraysAsPointers),
NubArrayType arr => CreateArrayType(arr, variableName),
NubFuncType fn => CreateFuncType(fn, variableName),
NubStructType st => $"struct {st.Module}_{st.Name}_{NameMangler.Mangle(st)}" + (variableName != null ? $" {variableName}" : ""),
_ => throw new NotSupportedException($"C type generation not supported for: {type}")
};
}
private static string CreateIntType(NubIntType intType, string? varName)
{
var cType = intType.Width switch
{
8 => intType.Signed ? "char" : "unsigned char",
16 => intType.Signed ? "short" : "unsigned short",
32 => intType.Signed ? "int" : "unsigned int",
64 => intType.Signed ? "long long" : "unsigned long long",
_ => throw new NotSupportedException($"Unsupported integer width: {intType.Width}")
};
return cType + (varName != null ? $" {varName}" : "");
}
private static string CreateFloatType(NubFloatType floatType, string? varName)
{
var cType = floatType.Width switch
{
32 => "float",
64 => "double",
_ => throw new NotSupportedException($"Unsupported float width: {floatType.Width}")
};
return cType + (varName != null ? $" {varName}" : "");
}
private static string CreatePointerType(NubPointerType ptr, string? varName)
{
var baseType = Create(ptr.BaseType);
return baseType + "*" + (varName != null ? $" {varName}" : "");
}
private static string CreateConstArrayType(NubConstArrayType arr, string? varName, bool inStructDef)
{
var elementType = Create(arr.ElementType);
// Treat const arrays as pointers unless in a struct definition
if (!inStructDef)
{
return elementType + "*" + (varName != null ? $" {varName}" : "");
}
if (varName != null)
{
return $"{elementType} {varName}[{arr.Size}]";
}
return $"{elementType}[{arr.Size}]";
}
private static string CreateArrayType(NubArrayType arr, string? varName)
{
var elementType = Create(arr.ElementType);
return elementType + "*" + (varName != null ? $" {varName}" : "");
}
private static string CreateFuncType(NubFuncType fn, string? varName)
{
var returnType = Create(fn.ReturnType);
var parameters = string.Join(", ", fn.Parameters.Select(p => Create(p)));
if (string.IsNullOrEmpty(parameters))
{
parameters = "void";
}
if (varName != null)
{
return $"{returnType} (*{varName})({parameters})";
}
return $"{returnType} (*)({parameters})";
}
}

View File

@@ -1,590 +0,0 @@
using System.Diagnostics;
using System.Text;
using NubLang.Ast;
using NubLang.Syntax;
namespace NubLang.Generation;
public class Generator
{
private readonly CompilationUnit _compilationUnit;
private readonly IndentedTextWriter _writer;
private readonly Stack<List<DeferNode>> _deferStack = [];
private int _tmpIndex;
public Generator(CompilationUnit compilationUnit)
{
_compilationUnit = compilationUnit;
_writer = new IndentedTextWriter();
}
// todo(nub31): Handle name collisions
private string NewTmp()
{
return $"_t{++_tmpIndex}";
}
private static string FuncName(string module, string name, string? externSymbol)
{
return externSymbol ?? $"{module}_{name}";
}
public string Emit()
{
_writer.WriteLine("""
struct nub_string
{
unsigned long long length;
char *data;
};
struct nub_slice
{
unsigned long long length;
void *data;
};
""");
foreach (var structType in _compilationUnit.ImportedStructTypes)
{
_writer.WriteLine(CType.Create(structType));
_writer.WriteLine("{");
using (_writer.Indent())
{
foreach (var field in structType.Fields)
{
_writer.WriteLine($"{CType.Create(field.Type, field.Name, constArraysAsPointers: false)};");
}
}
_writer.WriteLine("};");
_writer.WriteLine();
}
// note(nub31): Forward declarations
foreach (var prototype in _compilationUnit.ImportedFunctions)
{
EmitLine(prototype.Tokens.FirstOrDefault());
var parameters = prototype.Parameters.Count != 0
? string.Join(", ", prototype.Parameters.Select(x => CType.Create(x.Type, x.Name)))
: "void";
var name = FuncName(prototype.Module, prototype.Name, prototype.ExternSymbol);
_writer.WriteLine($"{CType.Create(prototype.ReturnType, name)}({parameters});");
_writer.WriteLine();
}
// note(nub31): Normal functions
foreach (var funcNode in _compilationUnit.Functions)
{
if (funcNode.Body == null) continue;
EmitLine(funcNode.Tokens.FirstOrDefault());
var parameters = funcNode.Prototype.Parameters.Count != 0
? string.Join(", ", funcNode.Prototype.Parameters.Select(x => CType.Create(x.Type, x.Name)))
: "void";
var name = FuncName(funcNode.Module, funcNode.Name, funcNode.Prototype.ExternSymbol);
_writer.WriteLine($"{CType.Create(funcNode.Prototype.ReturnType, name)}({parameters})");
_writer.WriteLine("{");
using (_writer.Indent())
{
EmitBlock(funcNode.Body);
}
_writer.WriteLine("}");
_writer.WriteLine();
}
return _writer.ToString();
}
private void EmitStatement(StatementNode statementNode)
{
EmitLine(statementNode.Tokens.FirstOrDefault());
switch (statementNode)
{
case AssignmentNode assignmentNode:
EmitAssignment(assignmentNode);
break;
case BlockNode blockNode:
_writer.WriteLine("{");
using (_writer.Indent())
{
EmitBlock(blockNode);
}
_writer.WriteLine("}");
break;
case BreakNode breakNode:
EmitBreak(breakNode);
break;
case ContinueNode continueNode:
EmitContinue(continueNode);
break;
case DeferNode deferNode:
EmitDefer(deferNode);
break;
case ForConstArrayNode forConstArrayNode:
EmitForConstArray(forConstArrayNode);
break;
case ForSliceNode forSliceNode:
EmitForSlice(forSliceNode);
break;
case IfNode ifNode:
EmitIf(ifNode);
break;
case ReturnNode returnNode:
EmitReturn(returnNode);
break;
case StatementFuncCallNode statementFuncCallNode:
EmitStatementFuncCall(statementFuncCallNode);
break;
case VariableDeclarationNode variableDeclarationNode:
EmitVariableDeclaration(variableDeclarationNode);
break;
case WhileNode whileNode:
EmitWhile(whileNode);
break;
default:
throw new ArgumentOutOfRangeException(nameof(statementNode));
}
}
private void EmitLine(Token? token)
{
if (token == null) return;
var file = token.Span.FilePath;
var line = token.Span.Start.Line;
_writer.WriteLine($"#line {line} \"{file}\"");
}
private void EmitAssignment(AssignmentNode assignmentNode)
{
var target = EmitExpression(assignmentNode.Target);
var value = EmitExpression(assignmentNode.Value);
_writer.WriteLine($"{target} = {value};");
}
private void EmitBreak(BreakNode _)
{
// todo(nub31): Emit deferred statements
_writer.WriteLine("break;");
}
private void EmitContinue(ContinueNode _)
{
// todo(nub31): Emit deferred statements
_writer.WriteLine("continue;");
}
private void EmitDefer(DeferNode deferNode)
{
_deferStack.Peek().Add(deferNode);
}
private void EmitForSlice(ForSliceNode forSliceNode)
{
var targetType = (NubSliceType)forSliceNode.Target.Type;
var target = EmitExpression(forSliceNode.Target);
var indexName = forSliceNode.IndexName ?? NewTmp();
_writer.WriteLine($"for (unsigned long long {indexName} = 0; {indexName} < {target}.length; ++{indexName})");
_writer.WriteLine("{");
using (_writer.Indent())
{
_writer.WriteLine($"{CType.Create(targetType.ElementType, forSliceNode.ElementName)} = (({CType.Create(targetType.ElementType)}*){target}.data)[{indexName}];");
EmitBlock(forSliceNode.Body);
}
_writer.WriteLine("}");
}
private void EmitForConstArray(ForConstArrayNode forConstArrayNode)
{
var targetType = (NubConstArrayType)forConstArrayNode.Target.Type;
var target = EmitExpression(forConstArrayNode.Target);
var indexName = forConstArrayNode.IndexName ?? NewTmp();
_writer.WriteLine($"for (unsigned long long {indexName} = 0; {indexName} < {targetType.Size}; ++{indexName})");
_writer.WriteLine("{");
using (_writer.Indent())
{
_writer.WriteLine($"{CType.Create(targetType.ElementType, forConstArrayNode.ElementName)} = {target}[{indexName}];");
EmitBlock(forConstArrayNode.Body);
}
_writer.WriteLine("}");
}
private void EmitIf(IfNode ifNode, bool elseIf = false)
{
var condition = EmitExpression(ifNode.Condition);
_writer.WriteLine($"{(elseIf ? "else " : "")}if ({condition})");
_writer.WriteLine("{");
using (_writer.Indent())
{
EmitBlock(ifNode.Body);
}
_writer.WriteLine("}");
ifNode.Else?.Match
(
elseIfNode => EmitIf(elseIfNode, true),
elseNode =>
{
_writer.WriteLine("else");
_writer.WriteLine("{");
using (_writer.Indent())
{
EmitBlock(elseNode);
}
_writer.WriteLine("}");
}
);
}
private void EmitReturn(ReturnNode returnNode)
{
if (returnNode.Value == null)
{
var blockDefers = _deferStack.Peek();
for (var i = blockDefers.Count - 1; i >= 0; i--)
{
EmitStatement(blockDefers[i].Statement);
}
_writer.WriteLine("return;");
}
else
{
var returnValue = EmitExpression(returnNode.Value);
if (_deferStack.Peek().Count != 0)
{
var tmp = NewTmp();
_writer.WriteLine($"{CType.Create(returnNode.Value.Type, tmp)} = {returnValue};");
var blockDefers = _deferStack.Peek();
for (var i = blockDefers.Count - 1; i >= 0; i--)
{
EmitStatement(blockDefers[i].Statement);
}
EmitLine(returnNode.Tokens.FirstOrDefault());
_writer.WriteLine($"return {tmp};");
}
else
{
EmitLine(returnNode.Tokens.FirstOrDefault());
_writer.WriteLine($"return {returnValue};");
}
}
}
private void EmitStatementFuncCall(StatementFuncCallNode statementFuncCallNode)
{
var funcCall = EmitFuncCall(statementFuncCallNode.FuncCall);
_writer.WriteLine($"{funcCall};");
}
private void EmitVariableDeclaration(VariableDeclarationNode variableDeclarationNode)
{
if (variableDeclarationNode.Assignment != null)
{
var value = EmitExpression(variableDeclarationNode.Assignment);
_writer.WriteLine($"{CType.Create(variableDeclarationNode.Type, variableDeclarationNode.Name)} = {value};");
}
else
{
_writer.WriteLine($"{CType.Create(variableDeclarationNode.Type, variableDeclarationNode.Name)};");
}
}
private void EmitWhile(WhileNode whileNode)
{
var condition = EmitExpression(whileNode.Condition);
_writer.WriteLine($"while ({condition})");
_writer.WriteLine("{");
using (_writer.Indent())
{
EmitBlock(whileNode.Body);
}
_writer.WriteLine("}");
}
private string EmitExpression(ExpressionNode expressionNode)
{
if (expressionNode is IntermediateExpression)
{
throw new UnreachableException("Type checker fucked up");
}
var expr = expressionNode switch
{
ArrayIndexAccessNode arrayIndexAccessNode => EmitArrayIndexAccess(arrayIndexAccessNode),
ArrayInitializerNode arrayInitializerNode => EmitArrayInitializer(arrayInitializerNode),
BinaryExpressionNode binaryExpressionNode => EmitBinaryExpression(binaryExpressionNode),
BoolLiteralNode boolLiteralNode => boolLiteralNode.Value ? "true" : "false",
ConstArrayIndexAccessNode constArrayIndexAccessNode => EmitConstArrayIndexAccess(constArrayIndexAccessNode),
ConstArrayInitializerNode constArrayInitializerNode => EmitConstArrayInitializer(constArrayInitializerNode),
CStringLiteralNode cStringLiteralNode => $"\"{cStringLiteralNode.Value}\"",
DereferenceNode dereferenceNode => EmitDereference(dereferenceNode),
Float32LiteralNode float32LiteralNode => EmitFloat32Literal(float32LiteralNode),
Float64LiteralNode float64LiteralNode => EmitFloat64Literal(float64LiteralNode),
CastNode castNode => EmitCast(castNode),
FuncCallNode funcCallNode => EmitFuncCall(funcCallNode),
FuncIdentifierNode funcIdentifierNode => FuncName(funcIdentifierNode.Module, funcIdentifierNode.Name, funcIdentifierNode.ExternSymbol),
AddressOfNode addressOfNode => EmitAddressOf(addressOfNode),
SizeNode sizeBuiltinNode => $"sizeof({CType.Create(sizeBuiltinNode.TargetType)})",
SliceIndexAccessNode sliceIndexAccessNode => EmitSliceArrayIndexAccess(sliceIndexAccessNode),
StringLiteralNode stringLiteralNode => EmitStringLiteral(stringLiteralNode),
StructFieldAccessNode structFieldAccessNode => EmitStructFieldAccess(structFieldAccessNode),
StructInitializerNode structInitializerNode => EmitStructInitializer(structInitializerNode),
I8LiteralNode i8LiteralNode => EmitI8Literal(i8LiteralNode),
I16LiteralNode i16LiteralNode => EmitI16Literal(i16LiteralNode),
I32LiteralNode i32LiteralNode => EmitI32Literal(i32LiteralNode),
I64LiteralNode i64LiteralNode => EmitI64Literal(i64LiteralNode),
U8LiteralNode u8LiteralNode => EmitU8Literal(u8LiteralNode),
U16LiteralNode u16LiteralNode => EmitU16Literal(u16LiteralNode),
U32LiteralNode u32LiteralNode => EmitU32Literal(u32LiteralNode),
U64LiteralNode u64LiteralNode => EmitU64Literal(u64LiteralNode),
UnaryExpressionNode unaryExpressionNode => EmitUnaryExpression(unaryExpressionNode),
VariableIdentifierNode variableIdentifierNode => variableIdentifierNode.Name,
_ => throw new ArgumentOutOfRangeException(nameof(expressionNode))
};
return $"({expr})";
}
private string EmitArrayIndexAccess(ArrayIndexAccessNode arrayIndexAccessNode)
{
var target = EmitExpression(arrayIndexAccessNode.Target);
var index = EmitExpression(arrayIndexAccessNode.Index);
return $"{target}[{index}]";
}
private string EmitArrayInitializer(ArrayInitializerNode arrayInitializerNode)
{
var values = new List<string>();
foreach (var value in arrayInitializerNode.Values)
{
values.Add(EmitExpression(value));
}
var arrayType = (NubArrayType)arrayInitializerNode.Type;
return $"({CType.Create(arrayType.ElementType)}[]){{{string.Join(", ", values)}}}";
}
private string EmitBinaryExpression(BinaryExpressionNode binaryExpressionNode)
{
var left = EmitExpression(binaryExpressionNode.Left);
var right = EmitExpression(binaryExpressionNode.Right);
var op = binaryExpressionNode.Operator switch
{
BinaryOperator.Plus => "+",
BinaryOperator.Minus => "-",
BinaryOperator.Multiply => "*",
BinaryOperator.Divide => "/",
BinaryOperator.Modulo => "%",
BinaryOperator.Equal => "==",
BinaryOperator.NotEqual => "!=",
BinaryOperator.LessThan => "<",
BinaryOperator.LessThanOrEqual => "<=",
BinaryOperator.GreaterThan => ">",
BinaryOperator.GreaterThanOrEqual => ">=",
BinaryOperator.LogicalAnd => "&&",
BinaryOperator.LogicalOr => "||",
BinaryOperator.BitwiseAnd => "&",
BinaryOperator.BitwiseOr => "|",
BinaryOperator.BitwiseXor => "^",
BinaryOperator.LeftShift => "<<",
BinaryOperator.RightShift => ">>",
_ => throw new ArgumentOutOfRangeException()
};
return $"{left} {op} {right}";
}
private string EmitConstArrayIndexAccess(ConstArrayIndexAccessNode constArrayIndexAccessNode)
{
var target = EmitExpression(constArrayIndexAccessNode.Target);
var index = EmitExpression(constArrayIndexAccessNode.Index);
// todo(nub31): We can emit bounds checking here
return $"{target}[{index}]";
}
private string EmitConstArrayInitializer(ConstArrayInitializerNode arrayInitializerNode)
{
var values = new List<string>();
foreach (var value in arrayInitializerNode.Values)
{
values.Add(EmitExpression(value));
}
var arrayType = (NubConstArrayType)arrayInitializerNode.Type;
return $"({CType.Create(arrayType.ElementType)}[{arrayType.Size}]){{{string.Join(", ", values)}}}";
}
private string EmitDereference(DereferenceNode dereferenceNode)
{
var pointer = EmitExpression(dereferenceNode.Target);
return $"*{pointer}";
}
private string EmitFloat32Literal(Float32LiteralNode float32LiteralNode)
{
var str = float32LiteralNode.Value.ToString("G9", System.Globalization.CultureInfo.InvariantCulture);
if (!str.Contains('.') && !str.Contains('e') && !str.Contains('E'))
{
str += ".0";
}
return str + "f";
}
private string EmitFloat64Literal(Float64LiteralNode float64LiteralNode)
{
var str = float64LiteralNode.Value.ToString("G17", System.Globalization.CultureInfo.InvariantCulture);
if (!str.Contains('.') && !str.Contains('e') && !str.Contains('E'))
{
str += ".0";
}
return str;
}
private string EmitCast(CastNode castNode)
{
var value = EmitExpression(castNode.Value);
if (castNode is { Type: NubSliceType sliceType, Value.Type: NubConstArrayType arrayType })
{
return $"({CType.Create(sliceType)}){{.length = {arrayType.Size}, .data = (void*){value}}}";
}
return $"({CType.Create(castNode.Type)}){value}";
}
private string EmitFuncCall(FuncCallNode funcCallNode)
{
var name = EmitExpression(funcCallNode.Expression);
var parameterNames = funcCallNode.Parameters.Select(EmitExpression).ToList();
return $"{name}({string.Join(", ", parameterNames)})";
}
private string EmitAddressOf(AddressOfNode addressOfNode)
{
var value = EmitExpression(addressOfNode.LValue);
return $"&{value}";
}
private string EmitSliceArrayIndexAccess(SliceIndexAccessNode sliceIndexAccessNode)
{
var targetType = (NubSliceType)sliceIndexAccessNode.Target.Type;
var target = EmitExpression(sliceIndexAccessNode.Target);
var index = EmitExpression(sliceIndexAccessNode.Index);
// todo(nub31): We can emit bounds checking here
return $"(({CType.Create(targetType.ElementType)}*){target}.data)[{index}]";
}
private string EmitStringLiteral(StringLiteralNode stringLiteralNode)
{
var length = Encoding.UTF8.GetByteCount(stringLiteralNode.Value);
return $"(nub_string){{.length = {length}, .data = \"{stringLiteralNode.Value}\"}}";
}
private string EmitStructFieldAccess(StructFieldAccessNode structFieldAccessNode)
{
var structExpr = EmitExpression(structFieldAccessNode.Target);
return $"{structExpr}.{structFieldAccessNode.Field}";
}
private string EmitStructInitializer(StructInitializerNode structInitializerNode)
{
var initValues = new List<string>();
foreach (var initializer in structInitializerNode.Initializers)
{
var value = EmitExpression(initializer.Value);
initValues.Add($".{initializer.Key} = {value}");
}
var initString = initValues.Count == 0
? "0"
: string.Join(", ", initValues);
return $"({CType.Create(structInitializerNode.Type)}){{{initString}}}";
}
private string EmitI8Literal(I8LiteralNode i8LiteralNode)
{
return i8LiteralNode.Value.ToString();
}
private string EmitI16Literal(I16LiteralNode i16LiteralNode)
{
return i16LiteralNode.Value.ToString();
}
private string EmitI32Literal(I32LiteralNode i32LiteralNode)
{
return i32LiteralNode.Value.ToString();
}
private string EmitI64Literal(I64LiteralNode i64LiteralNode)
{
return i64LiteralNode.Value + "LL";
}
private string EmitU8Literal(U8LiteralNode u8LiteralNode)
{
return u8LiteralNode.Value.ToString();
}
private string EmitU16Literal(U16LiteralNode u16LiteralNode)
{
return u16LiteralNode.Value.ToString();
}
private string EmitU32Literal(U32LiteralNode u32LiteralNode)
{
return u32LiteralNode.Value.ToString();
}
private string EmitU64Literal(U64LiteralNode u64LiteralNode)
{
return u64LiteralNode.Value + "ULL";
}
private string EmitUnaryExpression(UnaryExpressionNode unaryExpressionNode)
{
var value = EmitExpression(unaryExpressionNode.Operand);
return unaryExpressionNode.Operator switch
{
UnaryOperator.Negate => $"-{value}",
UnaryOperator.Invert => $"!{value}",
_ => throw new ArgumentOutOfRangeException()
};
}
private void EmitBlock(BlockNode blockNode)
{
_deferStack.Push([]);
foreach (var statementNode in blockNode.Statements)
{
EmitStatement(statementNode);
}
var blockDefers = _deferStack.Pop();
for (var i = blockDefers.Count - 1; i >= 0; i--)
{
EmitStatement(blockDefers[i].Statement);
}
}
}

View File

@@ -0,0 +1,781 @@
using System.Text;
using LLVMSharp.Interop;
using NubLang.Ast;
using NubLang.Modules;
using NubLang.Types;
namespace NubLang.Generation;
public class LlvmSharpGenerator
{
private string _module = string.Empty;
private LLVMContextRef _context;
private LLVMModuleRef _llvmModule;
private LLVMBuilderRef _builder;
private readonly Dictionary<string, LLVMTypeRef> _structTypes = new();
private readonly Dictionary<string, LLVMValueRef> _functions = new();
private readonly Dictionary<string, LLVMValueRef> _locals = new();
private readonly Stack<(LLVMBasicBlockRef breakBlock, LLVMBasicBlockRef continueBlock)> _loopStack = new();
public void Emit(List<TopLevelNode> topLevelNodes, ModuleRepository repository, string sourceFileName, string outputPath)
{
_module = topLevelNodes.OfType<ModuleNode>().First().NameToken.Value;
_context = LLVMContextRef.Global;
_llvmModule = _context.CreateModuleWithName(sourceFileName);
_llvmModule.Target = "x86_64-pc-linux-gnu";
_llvmModule.DataLayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128";
_builder = _context.CreateBuilder();
_structTypes.Clear();
_functions.Clear();
_locals.Clear();
_loopStack.Clear();
var stringType = _context.CreateNamedStruct("nub.string");
stringType.StructSetBody([LLVMTypeRef.Int64, LLVMTypeRef.CreatePointer(LLVMTypeRef.Int8, 0)], false);
_structTypes["nub.string"] = stringType;
foreach (var module in repository.GetAll())
{
foreach (var structType in module.StructTypes)
{
var structName = StructName(structType.Module, structType.Name);
var llvmStructType = _context.CreateNamedStruct(structName);
_structTypes[structName] = llvmStructType;
}
}
foreach (var module in repository.GetAll())
{
foreach (var structType in module.StructTypes)
{
var structName = StructName(structType.Module, structType.Name);
var llvmStructType = _structTypes[structName];
var fieldTypes = structType.Fields.Select(f => MapType(f.Type)).ToArray();
llvmStructType.StructSetBody(fieldTypes, false);
}
}
foreach (var module in repository.GetAll())
{
foreach (var prototype in module.FunctionPrototypes)
{
CreateFunctionDeclaration(prototype, module.Name);
}
}
foreach (var structNode in topLevelNodes.OfType<StructNode>())
{
EmitStructConstructor(structNode);
}
foreach (var funcNode in topLevelNodes.OfType<FuncNode>())
{
if (funcNode.Body != null)
{
EmitFunction(funcNode);
}
}
if (!_llvmModule.TryVerify(LLVMVerifierFailureAction.LLVMPrintMessageAction, out var error))
{
// throw new Exception($"LLVM module verification failed: {error}");
}
_llvmModule.PrintToFile(outputPath);
_builder.Dispose();
}
private void CreateFunctionDeclaration(FuncPrototypeNode prototype, string moduleName)
{
var funcName = FuncName(moduleName, prototype.NameToken.Value, prototype.ExternSymbolToken?.Value);
var paramTypes = prototype.Parameters.Select(p => MapType(p.Type)).ToArray();
var returnType = MapType(prototype.ReturnType);
var funcType = LLVMTypeRef.CreateFunction(returnType, paramTypes);
var func = _llvmModule.AddFunction(funcName, funcType);
func.FunctionCallConv = (uint)LLVMCallConv.LLVMCCallConv;
for (var i = 0; i < prototype.Parameters.Count; i++)
{
func.GetParam((uint)i).Name = prototype.Parameters[i].NameToken.Value;
}
_functions[funcName] = func;
}
private void EmitStructConstructor(StructNode structNode)
{
var structType = _structTypes[StructName(_module, structNode.NameToken.Value)];
var ptrType = LLVMTypeRef.CreatePointer(structType, 0);
var funcType = LLVMTypeRef.CreateFunction(LLVMTypeRef.Void, [ptrType]);
var funcName = StructConstructorName(_module, structNode.NameToken.Value);
var func = _llvmModule.AddFunction(funcName, funcType);
func.FunctionCallConv = (uint)LLVMCallConv.LLVMCCallConv;
var entryBlock = func.AppendBasicBlock("entry");
_builder.PositionAtEnd(entryBlock);
var selfParam = func.GetParam(0);
selfParam.Name = "self";
_locals.Clear();
foreach (var field in structNode.Fields)
{
if (field.Value != null)
{
var index = structNode.StructType.GetFieldIndex(field.NameToken.Value);
var fieldPtr = _builder.BuildStructGEP2(structType, selfParam, (uint)index);
EmitExpressionInto(field.Value, fieldPtr);
}
}
_builder.BuildRetVoid();
_functions[funcName] = func;
}
private void EmitFunction(FuncNode funcNode)
{
var funcName = FuncName(_module, funcNode.Prototype.NameToken.Value, funcNode.Prototype.ExternSymbolToken?.Value);
var func = _functions[funcName];
var entryBlock = func.AppendBasicBlock("entry");
_builder.PositionAtEnd(entryBlock);
_locals.Clear();
for (uint i = 0; i < funcNode.Prototype.Parameters.Count; i++)
{
var param = func.GetParam(i);
var paramNode = funcNode.Prototype.Parameters[(int)i];
var alloca = _builder.BuildAlloca(MapType(paramNode.Type), paramNode.NameToken.Value);
_builder.BuildStore(param, alloca);
_locals[paramNode.NameToken.Value] = alloca;
}
EmitBlock(funcNode.Body!);
if (funcNode.Prototype.ReturnType is NubVoidType)
{
if (_builder.InsertBlock.Terminator.Handle == IntPtr.Zero)
{
_builder.BuildRetVoid();
}
}
}
private void EmitBlock(BlockNode blockNode)
{
foreach (var statement in blockNode.Statements)
{
EmitStatement(statement);
}
}
private void EmitStatement(StatementNode statement)
{
switch (statement)
{
case AssignmentNode assignment:
EmitAssignment(assignment);
break;
case BlockNode block:
EmitBlock(block);
break;
case BreakNode:
EmitBreak();
break;
case ContinueNode:
EmitContinue();
break;
case IfNode ifNode:
EmitIf(ifNode);
break;
case ReturnNode returnNode:
EmitReturn(returnNode);
break;
case StatementFuncCallNode funcCall:
EmitExpression(funcCall.FuncCall);
break;
case VariableDeclarationNode varDecl:
EmitVariableDeclaration(varDecl);
break;
case WhileNode whileNode:
EmitWhile(whileNode);
break;
default:
throw new NotImplementedException($"Statement type {statement.GetType()} not implemented");
}
}
private void EmitAssignment(AssignmentNode assignment)
{
var targetPtr = EmitExpression(assignment.Target, asLValue: true);
var value = EmitExpression(assignment.Value);
_builder.BuildStore(value, targetPtr);
}
private void EmitBreak()
{
var (breakBlock, _) = _loopStack.Peek();
_builder.BuildBr(breakBlock);
}
private void EmitContinue()
{
var (_, continueBlock) = _loopStack.Peek();
_builder.BuildBr(continueBlock);
}
private void EmitIf(IfNode ifNode)
{
var condition = EmitExpression(ifNode.Condition);
var func = _builder.InsertBlock.Parent;
var thenBlock = func.AppendBasicBlock("if.then");
var elseBlock = ifNode.Else.HasValue ? func.AppendBasicBlock("if.else") : default;
var endBlock = func.AppendBasicBlock("if.end");
_builder.BuildCondBr(condition, thenBlock, ifNode.Else.HasValue ? elseBlock : endBlock);
_builder.PositionAtEnd(thenBlock);
EmitBlock(ifNode.Body);
if (_builder.InsertBlock.Terminator.Handle == IntPtr.Zero)
{
_builder.BuildBr(endBlock);
}
if (ifNode.Else.HasValue)
{
_builder.PositionAtEnd(elseBlock);
ifNode.Else.Value.Match(EmitIf, EmitBlock);
if (_builder.InsertBlock.Terminator.Handle == IntPtr.Zero)
{
_builder.BuildBr(endBlock);
}
}
_builder.PositionAtEnd(endBlock);
}
private void EmitReturn(ReturnNode returnNode)
{
if (returnNode.Value != null)
{
var value = EmitExpression(returnNode.Value);
_builder.BuildRet(value);
}
else
{
_builder.BuildRetVoid();
}
}
private void EmitVariableDeclaration(VariableDeclarationNode varDecl)
{
var alloca = _builder.BuildAlloca(MapType(varDecl.Type), varDecl.NameToken.Value);
_locals[varDecl.NameToken.Value] = alloca;
if (varDecl.Assignment != null)
{
EmitExpressionInto(varDecl.Assignment, alloca);
}
}
private void EmitWhile(WhileNode whileNode)
{
var func = _builder.InsertBlock.Parent;
var condBlock = func.AppendBasicBlock("while.cond");
var bodyBlock = func.AppendBasicBlock("while.body");
var endBlock = func.AppendBasicBlock("while.end");
_loopStack.Push((endBlock, condBlock));
_builder.BuildBr(condBlock);
_builder.PositionAtEnd(condBlock);
var condition = EmitExpression(whileNode.Condition);
_builder.BuildCondBr(condition, bodyBlock, endBlock);
_builder.PositionAtEnd(bodyBlock);
EmitBlock(whileNode.Body);
if (_builder.InsertBlock.Terminator.Handle == IntPtr.Zero)
{
_builder.BuildBr(condBlock);
}
_loopStack.Pop();
_builder.PositionAtEnd(endBlock);
}
private LLVMValueRef EmitExpression(ExpressionNode expr, bool asLValue = false)
{
var result = expr switch
{
StringLiteralNode stringLiteralNode => EmitStringLiteral(stringLiteralNode),
CStringLiteralNode cStringLiteralNode => EmitCStringLiteral(cStringLiteralNode),
BoolLiteralNode b => LLVMValueRef.CreateConstInt(LLVMTypeRef.Int1, b.Value ? 1UL : 0UL),
I8LiteralNode i => LLVMValueRef.CreateConstInt(LLVMTypeRef.Int8, (ulong)i.Value, true),
I16LiteralNode i => LLVMValueRef.CreateConstInt(LLVMTypeRef.Int16, (ulong)i.Value, true),
I32LiteralNode i => LLVMValueRef.CreateConstInt(LLVMTypeRef.Int32, (ulong)i.Value, true),
I64LiteralNode i => LLVMValueRef.CreateConstInt(LLVMTypeRef.Int64, (ulong)i.Value, true),
U8LiteralNode u => LLVMValueRef.CreateConstInt(LLVMTypeRef.Int8, u.Value),
U16LiteralNode u => LLVMValueRef.CreateConstInt(LLVMTypeRef.Int16, u.Value),
U32LiteralNode u => LLVMValueRef.CreateConstInt(LLVMTypeRef.Int32, u.Value),
U64LiteralNode u => LLVMValueRef.CreateConstInt(LLVMTypeRef.Int64, u.Value),
Float32LiteralNode f => LLVMValueRef.CreateConstReal(LLVMTypeRef.Float, f.Value),
Float64LiteralNode f => LLVMValueRef.CreateConstReal(LLVMTypeRef.Double, f.Value),
VariableIdentifierNode v => EmitVariableIdentifier(v),
LocalFuncIdentifierNode localFuncIdentifierNode => EmitLocalFuncIdentifier(localFuncIdentifierNode),
ModuleFuncIdentifierNode moduleFuncIdentifierNode => EmitModuleFuncIdentifier(moduleFuncIdentifierNode),
BinaryExpressionNode bin => EmitBinaryExpression(bin),
UnaryExpressionNode unary => EmitUnaryExpression(unary),
StructFieldAccessNode field => EmitStructFieldAccess(field),
ConstArrayIndexAccessNode arr => EmitConstArrayIndexAccess(arr),
SliceIndexAccessNode sliceIndexAccessNode => EmitSliceIndexAccess(sliceIndexAccessNode),
ArrayIndexAccessNode arrayIndexAccessNode => EmitArrayIndexAccess(arrayIndexAccessNode),
ConstArrayInitializerNode constArrayInitializerNode => EmitConstArrayInitializer(constArrayInitializerNode),
StructInitializerNode structInitializerNode => EmitStructInitializer(structInitializerNode),
AddressOfNode addr => EmitAddressOf(addr),
DereferenceNode deref => EmitDereference(deref),
FuncCallNode funcCall => EmitFuncCall(funcCall),
CastNode cast => EmitCast(cast),
SizeNode size => LLVMValueRef.CreateConstInt(LLVMTypeRef.Int64, size.TargetType.GetSize()),
_ => throw new ArgumentOutOfRangeException(nameof(expr), expr, null)
};
if (expr is LValue)
{
if (asLValue)
{
return result;
}
return _builder.BuildLoad2(MapType(expr.Type), result);
}
if (asLValue)
{
throw new InvalidOperationException($"Expression of type {expr.GetType().Name} is not an lvalue and cannot be used where an address is required");
}
return result;
}
private void EmitExpressionInto(ExpressionNode expr, LLVMValueRef destPtr)
{
switch (expr)
{
case StructInitializerNode structInit:
EmitStructInitializer(structInit, destPtr);
return;
case ConstArrayInitializerNode arrayInit:
EmitConstArrayInitializer(arrayInit, destPtr);
return;
default:
{
var result = EmitExpression(expr);
_builder.BuildStore(result, destPtr);
break;
}
}
}
private LLVMValueRef EmitStringLiteral(StringLiteralNode stringLiteralNode)
{
var strValue = stringLiteralNode.Value;
var length = (ulong)Encoding.UTF8.GetByteCount(strValue);
var globalStr = _builder.BuildGlobalStringPtr(strValue);
var llvmStringType = MapType(stringLiteralNode.Type);
var strAlloca = _builder.BuildAlloca(llvmStringType);
var lengthPtr = _builder.BuildStructGEP2(llvmStringType, strAlloca, 0);
var lengthConst = LLVMValueRef.CreateConstInt(LLVMTypeRef.Int64, length);
_builder.BuildStore(lengthConst, lengthPtr);
var dataPtr = _builder.BuildStructGEP2(llvmStringType, strAlloca, 1);
_builder.BuildStore(globalStr, dataPtr);
return _builder.BuildLoad2(llvmStringType, strAlloca);
}
private LLVMValueRef EmitCStringLiteral(CStringLiteralNode cStringLiteralNode)
{
return _builder.BuildGlobalStringPtr(cStringLiteralNode.Value);
}
private LLVMValueRef EmitVariableIdentifier(VariableIdentifierNode v)
{
return _locals[v.NameToken.Value];
}
private LLVMValueRef EmitLocalFuncIdentifier(LocalFuncIdentifierNode localFuncIdentifierNode)
{
return _functions[FuncName(_module, localFuncIdentifierNode.NameToken.Value, localFuncIdentifierNode.ExternSymbolToken?.Value)];
}
private LLVMValueRef EmitModuleFuncIdentifier(ModuleFuncIdentifierNode moduleFuncIdentifierNode)
{
return _functions[FuncName(moduleFuncIdentifierNode.ModuleToken.Value, moduleFuncIdentifierNode.NameToken.Value, moduleFuncIdentifierNode.ExternSymbolToken?.Value)];
}
private LLVMValueRef EmitBinaryExpression(BinaryExpressionNode bin)
{
var left = EmitExpression(bin.Left);
var right = EmitExpression(bin.Right);
var leftType = bin.Left.Type;
var result = bin.Operator switch
{
BinaryOperator.Plus when leftType is NubIntType => _builder.BuildAdd(left, right),
BinaryOperator.Plus when leftType is NubFloatType => _builder.BuildFAdd(left, right),
BinaryOperator.Minus when leftType is NubIntType => _builder.BuildSub(left, right),
BinaryOperator.Minus when leftType is NubFloatType => _builder.BuildFSub(left, right),
BinaryOperator.Multiply when leftType is NubIntType => _builder.BuildMul(left, right),
BinaryOperator.Multiply when leftType is NubFloatType => _builder.BuildFMul(left, right),
BinaryOperator.Divide when leftType is NubIntType intType => intType.Signed ? _builder.BuildSDiv(left, right) : _builder.BuildUDiv(left, right),
BinaryOperator.Divide when leftType is NubFloatType => _builder.BuildFDiv(left, right),
BinaryOperator.Modulo when leftType is NubIntType intType => intType.Signed ? _builder.BuildSRem(left, right) : _builder.BuildURem(left, right),
BinaryOperator.Modulo when leftType is NubFloatType => _builder.BuildFRem(left, right),
BinaryOperator.LogicalAnd => _builder.BuildAnd(left, right),
BinaryOperator.LogicalOr => _builder.BuildOr(left, right),
BinaryOperator.Equal when leftType is NubIntType or NubBoolType or NubPointerType => _builder.BuildICmp(LLVMIntPredicate.LLVMIntEQ, left, right),
BinaryOperator.Equal when leftType is NubFloatType => _builder.BuildFCmp(LLVMRealPredicate.LLVMRealOEQ, left, right),
BinaryOperator.NotEqual when leftType is NubIntType or NubBoolType or NubPointerType => _builder.BuildICmp(LLVMIntPredicate.LLVMIntNE, left, right),
BinaryOperator.NotEqual when leftType is NubFloatType => _builder.BuildFCmp(LLVMRealPredicate.LLVMRealONE, left, right),
BinaryOperator.GreaterThan when leftType is NubIntType intType => _builder.BuildICmp(intType.Signed ? LLVMIntPredicate.LLVMIntSGT : LLVMIntPredicate.LLVMIntUGT, left, right),
BinaryOperator.GreaterThan when leftType is NubFloatType => _builder.BuildFCmp(LLVMRealPredicate.LLVMRealOGT, left, right),
BinaryOperator.GreaterThanOrEqual when leftType is NubIntType intType => _builder.BuildICmp(intType.Signed ? LLVMIntPredicate.LLVMIntSGE : LLVMIntPredicate.LLVMIntUGE, left, right),
BinaryOperator.GreaterThanOrEqual when leftType is NubFloatType => _builder.BuildFCmp(LLVMRealPredicate.LLVMRealOGE, left, right),
BinaryOperator.LessThan when leftType is NubIntType intType => _builder.BuildICmp(intType.Signed ? LLVMIntPredicate.LLVMIntSLT : LLVMIntPredicate.LLVMIntULT, left, right),
BinaryOperator.LessThan when leftType is NubFloatType => _builder.BuildFCmp(LLVMRealPredicate.LLVMRealOLT, left, right),
BinaryOperator.LessThanOrEqual when leftType is NubIntType intType => _builder.BuildICmp(intType.Signed ? LLVMIntPredicate.LLVMIntSLE : LLVMIntPredicate.LLVMIntULE, left, right),
BinaryOperator.LessThanOrEqual when leftType is NubFloatType => _builder.BuildFCmp(LLVMRealPredicate.LLVMRealOLE, left, right),
BinaryOperator.LeftShift => _builder.BuildShl(left, right),
BinaryOperator.RightShift when leftType is NubIntType intType => intType.Signed ? _builder.BuildAShr(left, right) : _builder.BuildLShr(left, right),
BinaryOperator.BitwiseAnd => _builder.BuildAnd(left, right),
BinaryOperator.BitwiseXor => _builder.BuildXor(left, right),
BinaryOperator.BitwiseOr => _builder.BuildOr(left, right),
_ => throw new ArgumentOutOfRangeException(nameof(bin.Operator))
};
return result;
}
private LLVMValueRef EmitUnaryExpression(UnaryExpressionNode unary)
{
var operand = EmitExpression(unary.Operand);
var result = unary.Operator switch
{
UnaryOperator.Negate when unary.Operand.Type is NubIntType => _builder.BuildNeg(operand),
UnaryOperator.Negate when unary.Operand.Type is NubFloatType => _builder.BuildFNeg(operand),
UnaryOperator.Invert => _builder.BuildXor(operand, LLVMValueRef.CreateConstInt(LLVMTypeRef.Int1, 1)),
_ => throw new ArgumentOutOfRangeException(nameof(unary.Operator))
};
return result;
}
private LLVMValueRef EmitFuncCall(FuncCallNode funcCall)
{
var funcPtr = EmitExpression(funcCall.Expression);
var args = funcCall.Parameters.Select(x => EmitExpression(x)).ToArray();
return _builder.BuildCall2(MapType(funcCall.Expression.Type), funcPtr, args, funcCall.Type is NubVoidType ? "" : "call");
}
private LLVMValueRef EmitStructFieldAccess(StructFieldAccessNode field)
{
var target = EmitExpression(field.Target, asLValue: true);
var structType = (NubStructType)field.Target.Type;
var index = structType.GetFieldIndex(field.FieldToken.Value);
var llvmStructType = _structTypes[StructName(structType.Module, structType.Name)];
return _builder.BuildStructGEP2(llvmStructType, target, (uint)index);
}
private LLVMValueRef EmitConstArrayIndexAccess(ConstArrayIndexAccessNode constArrayIndexAccessNode)
{
var arrayPtr = EmitExpression(constArrayIndexAccessNode.Target, asLValue: true);
var index = EmitExpression(constArrayIndexAccessNode.Index);
var indices = new[] { LLVMValueRef.CreateConstInt(LLVMTypeRef.Int32, 0), index };
return _builder.BuildInBoundsGEP2(MapType(constArrayIndexAccessNode.Target.Type), arrayPtr, indices);
}
private LLVMValueRef EmitSliceIndexAccess(SliceIndexAccessNode sliceIndexAccessNode)
{
var slicePtr = EmitExpression(sliceIndexAccessNode.Target, asLValue: true);
var index = EmitExpression(sliceIndexAccessNode.Index);
var sliceType = (NubSliceType)sliceIndexAccessNode.Target.Type;
var llvmSliceType = MapType(sliceType);
var elementType = MapType(sliceType.ElementType);
var dataPtrPtr = _builder.BuildStructGEP2(llvmSliceType, slicePtr, 1);
var dataPtr = _builder.BuildLoad2(LLVMTypeRef.CreatePointer(elementType, 0), dataPtrPtr);
return _builder.BuildInBoundsGEP2(elementType, dataPtr, [index]);
}
private LLVMValueRef EmitArrayIndexAccess(ArrayIndexAccessNode arrayIndexAccessNode)
{
var arrayPtr = EmitExpression(arrayIndexAccessNode.Target);
var index = EmitExpression(arrayIndexAccessNode.Index);
return _builder.BuildGEP2(MapType(arrayIndexAccessNode.Target.Type), arrayPtr, [index]);
}
private LLVMValueRef EmitConstArrayInitializer(ConstArrayInitializerNode constArrayInitializerNode, LLVMValueRef? destination = null)
{
var arrayType = (NubConstArrayType)constArrayInitializerNode.Type;
var llvmType = MapType(arrayType);
destination ??= _builder.BuildAlloca(llvmType);
for (var i = 0; i < constArrayInitializerNode.Values.Count; i++)
{
var indices = new[]
{
LLVMValueRef.CreateConstInt(LLVMTypeRef.Int32, 0),
LLVMValueRef.CreateConstInt(LLVMTypeRef.Int32, (ulong)i)
};
var elementPtr = _builder.BuildInBoundsGEP2(llvmType, destination.Value, indices);
EmitExpressionInto(constArrayInitializerNode.Values[i], elementPtr);
}
return destination.Value;
}
private LLVMValueRef EmitStructInitializer(StructInitializerNode structInitializerNode, LLVMValueRef? destination = null)
{
var type = (NubStructType)structInitializerNode.Type;
var llvmType = MapType(type);
destination ??= _builder.BuildAlloca(llvmType);
var constructorType = LLVMTypeRef.CreateFunction(LLVMTypeRef.Void, [LLVMTypeRef.CreatePointer(_structTypes[StructName(type.Module, type.Name)], 0)]);
var constructor = _functions[StructConstructorName(type.Module, type.Name)];
_builder.BuildCall2(constructorType, constructor, [destination.Value]);
foreach (var (name, value) in structInitializerNode.Initializers)
{
var fieldIndex = type.GetFieldIndex(name.Value);
var fieldPtr = _builder.BuildStructGEP2(llvmType, destination.Value, (uint)fieldIndex);
EmitExpressionInto(value, fieldPtr);
}
return destination.Value;
}
private LLVMValueRef EmitAddressOf(AddressOfNode addr)
{
return EmitExpression(addr.Target, asLValue: true);
}
private LLVMValueRef EmitDereference(DereferenceNode deref)
{
return EmitExpression(deref.Target, asLValue: false);
}
private LLVMValueRef EmitCast(CastNode castNode)
{
return castNode.ConversionType switch
{
CastNode.Conversion.IntToInt => EmitIntToIntCast(castNode),
CastNode.Conversion.FloatToFloat => EmitFloatToFloatCast(castNode),
CastNode.Conversion.IntToFloat => EmitIntToFloatCast(castNode),
CastNode.Conversion.FloatToInt => EmitFloatToIntCast(castNode),
CastNode.Conversion.PointerToPointer or CastNode.Conversion.PointerToUInt64 or CastNode.Conversion.UInt64ToPointer => _builder.BuildIntToPtr(EmitExpression(castNode.Value), MapType(castNode.Type)),
CastNode.Conversion.ConstArrayToSlice => EmitConstArrayToSliceCast(castNode),
CastNode.Conversion.ConstArrayToArray => EmitConstArrayToArrayCast(castNode),
CastNode.Conversion.StringToCString => EmitStringToCStringCast(castNode),
_ => throw new ArgumentOutOfRangeException(nameof(castNode.ConversionType))
};
}
private LLVMValueRef EmitIntToIntCast(CastNode castNode)
{
var sourceInt = (NubIntType)castNode.Value.Type;
var targetInt = (NubIntType)castNode.Type;
var source = EmitExpression(castNode.Value);
if (sourceInt.Width < targetInt.Width)
{
return sourceInt.Signed
? _builder.BuildSExt(source, MapType(targetInt))
: _builder.BuildZExt(source, MapType(targetInt));
}
if (sourceInt.Width > targetInt.Width)
{
return _builder.BuildTrunc(source, MapType(targetInt));
}
return _builder.BuildBitCast(source, MapType(targetInt));
}
private LLVMValueRef EmitFloatToFloatCast(CastNode castNode)
{
var sourceFloat = (NubFloatType)castNode.Value.Type;
var targetFloat = (NubFloatType)castNode.Type;
var source = EmitExpression(castNode.Value);
return sourceFloat.Width < targetFloat.Width
? _builder.BuildFPExt(source, MapType(castNode.Type))
: _builder.BuildFPTrunc(source, MapType(castNode.Type));
}
private LLVMValueRef EmitIntToFloatCast(CastNode castNode)
{
var sourceInt = (NubIntType)castNode.Value.Type;
var source = EmitExpression(castNode.Value);
return sourceInt.Signed
? _builder.BuildSIToFP(source, MapType(castNode.Type))
: _builder.BuildUIToFP(source, MapType(castNode.Type));
}
private LLVMValueRef EmitFloatToIntCast(CastNode castNode)
{
var targetInt = (NubIntType)castNode.Type;
var source = EmitExpression(castNode.Value);
return targetInt.Signed
? _builder.BuildFPToSI(source, MapType(targetInt))
: _builder.BuildFPToUI(source, MapType(targetInt));
}
private LLVMValueRef EmitConstArrayToSliceCast(CastNode castNode)
{
var sourceArrayType = (NubConstArrayType)castNode.Value.Type;
var targetSliceType = (NubSliceType)castNode.Type;
var source = EmitExpression(castNode.Value, asLValue: true);
var llvmArrayType = MapType(sourceArrayType);
var llvmSliceType = MapType(targetSliceType);
var indices = new[]
{
LLVMValueRef.CreateConstInt(LLVMTypeRef.Int32, 0),
LLVMValueRef.CreateConstInt(LLVMTypeRef.Int32, 0)
};
var firstElementPtr = _builder.BuildInBoundsGEP2(llvmArrayType, source, indices);
var slicePtr = _builder.BuildAlloca(llvmSliceType);
var lengthPtr = _builder.BuildStructGEP2(llvmSliceType, slicePtr, 0);
var length = LLVMValueRef.CreateConstInt(LLVMTypeRef.Int64, sourceArrayType.Size);
_builder.BuildStore(length, lengthPtr);
var dataPtrPtr = _builder.BuildStructGEP2(llvmSliceType, slicePtr, 1);
_builder.BuildStore(firstElementPtr, dataPtrPtr);
return _builder.BuildLoad2(llvmSliceType, slicePtr);
}
private LLVMValueRef EmitConstArrayToArrayCast(CastNode castNode)
{
var sourceArrayType = (NubConstArrayType)castNode.Value.Type;
var source = EmitExpression(castNode.Value, asLValue: true);
var indices = new[]
{
LLVMValueRef.CreateConstInt(LLVMTypeRef.Int32, 0),
LLVMValueRef.CreateConstInt(LLVMTypeRef.Int32, 0)
};
return _builder.BuildInBoundsGEP2(MapType(sourceArrayType), source, indices);
}
private LLVMValueRef EmitStringToCStringCast(CastNode castNode)
{
var source = EmitExpression(castNode.Value, asLValue: true);
var dataPtrPtr = _builder.BuildStructGEP2(MapType(castNode.Value.Type), source, 1);
return _builder.BuildLoad2(LLVMTypeRef.CreatePointer(LLVMTypeRef.Int8, 0), dataPtrPtr);
}
private LLVMTypeRef MapType(NubType type)
{
return type switch
{
NubBoolType => LLVMTypeRef.Int1,
NubIntType intType => LLVMTypeRef.CreateInt((uint)intType.Width),
NubFloatType floatType => floatType.Width == 32 ? LLVMTypeRef.Float : LLVMTypeRef.Double,
NubFuncType funcType => LLVMTypeRef.CreateFunction(MapType(funcType.ReturnType), funcType.Parameters.Select(MapType).ToArray()),
NubPointerType ptrType => LLVMTypeRef.CreatePointer(MapType(ptrType.BaseType), 0),
NubSliceType nubSliceType => MapSliceType(nubSliceType),
NubStringType => _structTypes["nub.string"],
NubArrayType arrType => LLVMTypeRef.CreatePointer(MapType(arrType.ElementType), 0),
NubConstArrayType constArr => LLVMTypeRef.CreateArray(MapType(constArr.ElementType), (uint)constArr.Size),
NubStructType structType => _structTypes[StructName(structType.Module, structType.Name)],
NubVoidType => LLVMTypeRef.Void,
_ => throw new ArgumentOutOfRangeException(nameof(type), type, null)
};
}
private LLVMTypeRef MapSliceType(NubSliceType nubSliceType)
{
var mangledName = NameMangler.Mangle(nubSliceType.ElementType);
var name = $"nub.slice.{mangledName}";
if (!_structTypes.TryGetValue(name, out var type))
{
type = _context.CreateNamedStruct(name);
type.StructSetBody([LLVMTypeRef.Int64, LLVMTypeRef.CreatePointer(MapType(nubSliceType.ElementType), 0)], false);
_structTypes[name] = type;
}
return type;
}
private static string StructName(string module, string name)
{
return $"struct.{module}.{name}";
}
private static string StructConstructorName(string module, string name)
{
return $"{StructName(module, name)}.new";
}
private static string FuncName(string module, string name, string? externSymbol)
{
if (externSymbol != null)
{
return externSymbol;
}
return $"{module}.{name}";
}
}

View File

@@ -0,0 +1,308 @@
using System.Diagnostics.CodeAnalysis;
using NubLang.Ast;
using NubLang.Diagnostics;
using NubLang.Syntax;
using NubLang.Types;
namespace NubLang.Modules;
public sealed class ModuleRepository
{
public static ModuleRepository Create(List<SyntaxTree> syntaxTrees)
{
var structTypes = new Dictionary<(string module, string name), NubStructType>();
var enumTypes = new Dictionary<(string module, string name), NubIntType>();
foreach (var syntaxTree in syntaxTrees)
{
var module = syntaxTree.TopLevelSyntaxNodes.OfType<ModuleSyntax>().FirstOrDefault();
if (module == null)
{
throw new CompileException(Diagnostic.Error("Module declaration missing").WithHelp("module \"main\"").Build());
}
foreach (var structSyntax in syntaxTree.TopLevelSyntaxNodes.OfType<StructSyntax>())
{
// note(nub31): Since not all struct types are registered yet, we cannot register field types as they might reference unregistered structs
var key = (module.NameToken.Value, structSyntax.NameToken.Value);
structTypes.Add(key, new NubStructType(module.NameToken.Value, structSyntax.NameToken.Value, structSyntax.Packed, []));
}
foreach (var enumSyntax in syntaxTree.TopLevelSyntaxNodes.OfType<EnumSyntax>())
{
NubIntType? underlyingType = null;
if (enumSyntax.Type != null)
{
if (enumSyntax.Type is not IntTypeSyntax intType)
{
throw new CompileException(Diagnostic.Error("Underlying type of enum must be an integer type").At(enumSyntax.Type).Build());
}
underlyingType = new NubIntType(intType.Signed, intType.Width);
}
underlyingType ??= new NubIntType(false, 64);
var key = (module.NameToken.Value, enumSyntax.NameToken.Value);
enumTypes.Add(key, underlyingType);
}
}
// note(nub31): Since all struct types are now registered, we can safely resolve the field types
foreach (var syntaxTree in syntaxTrees)
{
var module = syntaxTree.TopLevelSyntaxNodes.OfType<ModuleSyntax>().FirstOrDefault();
if (module == null)
{
throw new CompileException(Diagnostic.Error("Module declaration missing").WithHelp("module \"main\"").Build());
}
foreach (var structSyntax in syntaxTree.TopLevelSyntaxNodes.OfType<StructSyntax>())
{
var key = (module.NameToken.Value, structSyntax.NameToken.Value);
structTypes[key].Fields = structSyntax.Fields
.Select(x => new NubStructFieldType(x.NameToken.Value, ResolveType(x.Type, module.NameToken.Value), x.Value != null))
.ToList();
}
}
var modules = new Dictionary<string, Module>();
foreach (var syntaxTree in syntaxTrees)
{
var moduleDecl = syntaxTree.TopLevelSyntaxNodes.OfType<ModuleSyntax>().FirstOrDefault();
if (moduleDecl == null)
{
throw new CompileException(Diagnostic.Error("Module declaration missing").WithHelp("module \"main\"").Build());
}
var functionPrototypes = new List<FuncPrototypeNode>();
foreach (var funcSyntax in syntaxTree.TopLevelSyntaxNodes.OfType<FuncSyntax>())
{
var returnType = ResolveType(funcSyntax.Prototype.ReturnType, moduleDecl.NameToken.Value);
var parameters = funcSyntax.Prototype.Parameters.Select(x => new FuncParameterNode(x.Tokens, x.NameToken, ResolveType(x.Type, moduleDecl.NameToken.Value))).ToList();
functionPrototypes.Add(new FuncPrototypeNode(funcSyntax.Prototype.Tokens, funcSyntax.Prototype.NameToken, funcSyntax.Prototype.ExternSymbolToken, parameters, returnType));
}
var module = new Module
{
Name = moduleDecl.NameToken.Value,
StructTypes = structTypes.Where(x => x.Key.module == moduleDecl.NameToken.Value).Select(x => x.Value).ToList(),
EnumTypes = enumTypes
.Where(x => x.Key.module == moduleDecl.NameToken.Value)
.ToDictionary(x => x.Key.name, x => x.Value),
FunctionPrototypes = functionPrototypes
};
modules.Add(moduleDecl.NameToken.Value, module);
}
return new ModuleRepository(modules);
NubType ResolveType(TypeSyntax type, string currentModule)
{
return type switch
{
ArrayTypeSyntax arr => new NubArrayType(ResolveType(arr.BaseType, currentModule)),
BoolTypeSyntax => new NubBoolType(),
IntTypeSyntax i => new NubIntType(i.Signed, i.Width),
FloatTypeSyntax f => new NubFloatType(f.Width),
FuncTypeSyntax func => new NubFuncType(func.Parameters.Select(x => ResolveType(x, currentModule)).ToList(), ResolveType(func.ReturnType, currentModule)),
SliceTypeSyntax slice => new NubSliceType(ResolveType(slice.BaseType, currentModule)),
ConstArrayTypeSyntax arr => new NubConstArrayType(ResolveType(arr.BaseType, currentModule), arr.Size),
PointerTypeSyntax ptr => new NubPointerType(ResolveType(ptr.BaseType, currentModule)),
StringTypeSyntax => new NubStringType(),
CustomTypeSyntax c => ResolveCustomType(c, currentModule),
VoidTypeSyntax => new NubVoidType(),
_ => throw new NotSupportedException($"Unknown type syntax: {type}")
};
}
NubType ResolveCustomType(CustomTypeSyntax customType, string currentModule)
{
var customTypeKey = (customType.ModuleToken?.Value ?? currentModule, customType.NameToken.Value);
var resolvedStructType = structTypes.GetValueOrDefault(customTypeKey);
if (resolvedStructType != null)
{
return resolvedStructType;
}
var resolvedEnumType = enumTypes.GetValueOrDefault(customTypeKey);
if (resolvedEnumType != null)
{
return resolvedEnumType;
}
throw new CompileException(Diagnostic
.Error($"Type {customType.NameToken.Value} not found in module {customType.ModuleToken?.Value ?? currentModule}")
.At(customType)
.Build());
}
}
public ModuleRepository(Dictionary<string, Module> modules)
{
_modules = modules;
}
private readonly Dictionary<string, Module> _modules;
public Module Get(IdentifierToken ident)
{
var module = _modules.GetValueOrDefault(ident.Value);
if (module == null)
{
throw new CompileException(Diagnostic.Error($"Module {ident.Value} was not found").At(ident).Build());
}
return module;
}
public bool TryGet(IdentifierToken ident, [NotNullWhen(true)] out Module? module)
{
module = _modules.GetValueOrDefault(ident.Value);
return module != null;
}
public bool TryGet(string name, [NotNullWhen(true)] out Module? module)
{
module = _modules.GetValueOrDefault(name);
return module != null;
}
public List<Module> GetAll()
{
return _modules.Values.ToList();
}
public sealed class Module
{
public required string Name { get; init; }
public required List<FuncPrototypeNode> FunctionPrototypes { get; init; } = [];
public required List<NubStructType> StructTypes { get; init; } = [];
public required Dictionary<string, NubIntType> EnumTypes { get; init; } = [];
public bool TryResolveFunc(string name, [NotNullWhen(true)] out FuncPrototypeNode? value, [NotNullWhen(false)] out Diagnostic? diagnostic)
{
value = FunctionPrototypes.FirstOrDefault(x => x.NameToken.Value == name);
if (value == null)
{
value = null;
diagnostic = Diagnostic.Error($"Func {name} not found in module {Name}").Build();
return false;
}
diagnostic = null;
return true;
}
public bool TryResolveFunc(IdentifierToken name, [NotNullWhen(true)] out FuncPrototypeNode? value, [NotNullWhen(false)] out Diagnostic? diagnostic)
{
value = FunctionPrototypes.FirstOrDefault(x => x.NameToken.Value == name.Value);
if (value == null)
{
value = null;
diagnostic = Diagnostic.Error($"Func {name.Value} not found in module {Name}").At(name).Build();
return false;
}
diagnostic = null;
return true;
}
public FuncPrototypeNode ResolveFunc(IdentifierToken name)
{
if (!TryResolveFunc(name, out var value, out var diagnostic))
{
throw new CompileException(diagnostic);
}
return value;
}
public bool TryResolveStruct(string name, [NotNullWhen(true)] out NubStructType? value, [NotNullWhen(false)] out Diagnostic? diagnostic)
{
value = StructTypes.FirstOrDefault(x => x.Name == name);
if (value == null)
{
value = null;
diagnostic = Diagnostic.Error($"Struct {name} not found in module {Name}").Build();
return false;
}
diagnostic = null;
return true;
}
public bool TryResolveStruct(IdentifierToken name, [NotNullWhen(true)] out NubStructType? value, [NotNullWhen(false)] out Diagnostic? diagnostic)
{
value = StructTypes.FirstOrDefault(x => x.Name == name.Value);
if (value == null)
{
value = null;
diagnostic = Diagnostic.Error($"Struct {name.Value} not found in module {Name}").At(name).Build();
return false;
}
diagnostic = null;
return true;
}
public NubStructType ResolveStruct(IdentifierToken name)
{
if (!TryResolveStruct(name, out var value, out var diagnostic))
{
throw new CompileException(diagnostic);
}
return value;
}
public bool TryResolveEnum(string name, [NotNullWhen(true)] out NubIntType? value, [NotNullWhen(false)] out Diagnostic? diagnostic)
{
value = EnumTypes.GetValueOrDefault(name);
if (value == null)
{
value = null;
diagnostic = Diagnostic.Error($"Enum {name} not found in module {Name}").Build();
return false;
}
diagnostic = null;
return true;
}
public bool TryResolveEnum(IdentifierToken name, [NotNullWhen(true)] out NubIntType? value, [NotNullWhen(false)] out Diagnostic? diagnostic)
{
value = EnumTypes.GetValueOrDefault(name.Value);
if (value == null)
{
value = null;
diagnostic = Diagnostic.Error($"Enum {name.Value} not found in module {Name}").At(name).Build();
return false;
}
diagnostic = null;
return true;
}
public NubIntType ResolveEnum(IdentifierToken name)
{
if (!TryResolveEnum(name, out var value, out var diagnostic))
{
throw new CompileException(diagnostic);
}
return value;
}
}
}

View File

@@ -7,4 +7,8 @@
<IsAotCompatible>true</IsAotCompatible> <IsAotCompatible>true</IsAotCompatible>
</PropertyGroup> </PropertyGroup>
<ItemGroup>
<PackageReference Include="LLVMSharp" Version="20.1.2" />
</ItemGroup>
</Project> </Project>

View File

@@ -1,47 +0,0 @@
namespace NubLang.Syntax;
public sealed class Module
{
public static Dictionary<string, Module> Collect(List<SyntaxTree> syntaxTrees)
{
var modules = new Dictionary<string, Module>();
foreach (var syntaxTree in syntaxTrees)
{
if (!modules.TryGetValue(syntaxTree.ModuleName, out var module))
{
module = new Module();
modules.Add(syntaxTree.ModuleName, module);
}
module._definitions.AddRange(syntaxTree.Definitions);
}
return modules;
}
private readonly List<DefinitionSyntax> _definitions = [];
public List<StructSyntax> Structs(bool includePrivate)
{
return _definitions
.OfType<StructSyntax>()
.Where(x => x.Exported || includePrivate)
.ToList();
}
public List<FuncSyntax> Functions(bool includePrivate)
{
return _definitions
.OfType<FuncSyntax>()
.Where(x => x.Exported || includePrivate)
.ToList();
}
public List<EnumSyntax> Enums(bool includePrivate)
{
return _definitions
.OfType<EnumSyntax>()
.Where(x => x.Exported || includePrivate)
.ToList();
}
}

View File

@@ -9,19 +9,31 @@ public sealed class Parser
private int _tokenIndex; private int _tokenIndex;
private Token? CurrentToken => _tokenIndex < _tokens.Count ? _tokens[_tokenIndex] : null; private Token? CurrentToken => _tokenIndex < _tokens.Count ? _tokens[_tokenIndex] : null;
private bool HasTrailingWhitespace(Token token)
{
var index = _tokens.IndexOf(token);
return index + 1 < _tokens.Count && _tokens[index + 1] is WhitespaceToken or CommentToken;
}
private bool HasLeadingWhitespace(Token token)
{
var index = _tokens.IndexOf(token);
return index - 1 < _tokens.Count && _tokens[index - 1] is WhitespaceToken or CommentToken;
}
private bool HasToken => CurrentToken != null; private bool HasToken => CurrentToken != null;
public List<Diagnostic> Diagnostics { get; } = []; public List<Diagnostic> Diagnostics { get; set; } = [];
public SyntaxTree Parse(List<Token> tokens) public SyntaxTree Parse(List<Token> tokens)
{ {
Diagnostics.Clear();
_tokens = tokens; _tokens = tokens;
_tokenIndex = 0; _tokenIndex = 0;
string? moduleName = null; Diagnostics = [];
var imports = new List<string>();
var definitions = new List<DefinitionSyntax>(); var topLevelSyntaxNodes = new List<TopLevelSyntaxNode>();
while (HasToken) while (HasToken)
{ {
@@ -29,66 +41,34 @@ public sealed class Parser
{ {
var startIndex = _tokenIndex; var startIndex = _tokenIndex;
if (TryExpectSymbol(Symbol.Import))
{
var name = ExpectStringLiteral();
if (imports.Contains(name.Value))
{
Diagnostics.Add(Diagnostic
.Warning($"Module {name.Value} is imported twice")
.At(name)
.WithHelp($"Remove duplicate import \"{name.Value}\"")
.Build());
}
else
{
imports.Add(name.Value);
}
continue;
}
if (TryExpectSymbol(Symbol.Module))
{
if (moduleName != null)
{
throw new ParseException(Diagnostic
.Error("Module is declared more than once")
.At(CurrentToken)
.WithHelp("Remove duplicate module declaration")
.Build());
}
moduleName = ExpectStringLiteral().Value;
continue;
}
var exported = TryExpectSymbol(Symbol.Export); var exported = TryExpectSymbol(Symbol.Export);
var packed = TryExpectSymbol(Symbol.Packed);
if (TryExpectSymbol(Symbol.Extern)) if (TryExpectSymbol(Symbol.Extern))
{ {
var externSymbol = ExpectStringLiteral(); var externSymbol = ExpectStringLiteral();
ExpectSymbol(Symbol.Func); ExpectSymbol(Symbol.Func);
definitions.Add(ParseFunc(startIndex, exported, externSymbol.Value)); topLevelSyntaxNodes.Add(ParseFunc(startIndex, exported, externSymbol));
continue; continue;
} }
var keyword = ExpectSymbol(); var keyword = ExpectSymbol();
DefinitionSyntax definition = keyword.Symbol switch TopLevelSyntaxNode definition = keyword.Symbol switch
{ {
Symbol.Module => ParseModule(startIndex),
Symbol.Func => ParseFunc(startIndex, exported, null), Symbol.Func => ParseFunc(startIndex, exported, null),
Symbol.Struct => ParseStruct(startIndex, exported), Symbol.Struct => ParseStruct(startIndex, exported, packed),
Symbol.Enum => ParseEnum(startIndex, exported), Symbol.Enum => ParseEnum(startIndex, exported),
_ => throw new ParseException(Diagnostic _ => throw new CompileException(Diagnostic
.Error($"Expected 'func', 'struct', 'enum', 'import' or 'module' but found '{keyword.Symbol}'") .Error($"Expected 'func', 'struct', 'enum', 'import' or 'module' but found '{keyword.Symbol}'")
.WithHelp("Valid top level statements are 'func', 'struct', 'enum', 'import' and 'module'") .WithHelp("Valid top level statements are 'func', 'struct', 'enum', 'import' and 'module'")
.At(keyword) .At(keyword, _tokens)
.Build()) .Build())
}; };
definitions.Add(definition); topLevelSyntaxNodes.Add(definition);
} }
catch (ParseException e) catch (CompileException e)
{ {
Diagnostics.Add(e.Diagnostic); Diagnostics.Add(e.Diagnostic);
while (HasToken) while (HasToken)
@@ -103,7 +83,13 @@ public sealed class Parser
} }
} }
return new SyntaxTree(definitions, moduleName ?? "default", imports); return new SyntaxTree(topLevelSyntaxNodes, _tokens);
}
private ModuleSyntax ParseModule(int startIndex)
{
var name = ExpectIdentifier();
return new ModuleSyntax(GetTokens(startIndex), name);
} }
private FuncParameterSyntax ParseFuncParameter() private FuncParameterSyntax ParseFuncParameter()
@@ -113,10 +99,10 @@ public sealed class Parser
ExpectSymbol(Symbol.Colon); ExpectSymbol(Symbol.Colon);
var type = ParseType(); var type = ParseType();
return new FuncParameterSyntax(GetTokens(startIndex), name.Value, type); return new FuncParameterSyntax(GetTokens(startIndex), name, type);
} }
private FuncSyntax ParseFunc(int startIndex, bool exported, string? externSymbol) private FuncSyntax ParseFunc(int startIndex, bool exported, StringLiteralToken? externSymbol)
{ {
var name = ExpectIdentifier(); var name = ExpectIdentifier();
List<FuncParameterSyntax> parameters = []; List<FuncParameterSyntax> parameters = [];
@@ -136,7 +122,7 @@ public sealed class Parser
var returnType = TryExpectSymbol(Symbol.Colon) ? ParseType() : new VoidTypeSyntax([]); var returnType = TryExpectSymbol(Symbol.Colon) ? ParseType() : new VoidTypeSyntax([]);
var prototype = new FuncPrototypeSyntax(GetTokens(startIndex), name.Value, exported, externSymbol, parameters, returnType); var prototype = new FuncPrototypeSyntax(GetTokens(startIndex), name, exported, externSymbol, parameters, returnType);
BlockSyntax? body = null; BlockSyntax? body = null;
var bodyStartIndex = _tokenIndex; var bodyStartIndex = _tokenIndex;
@@ -148,7 +134,7 @@ public sealed class Parser
return new FuncSyntax(GetTokens(startIndex), prototype, body); return new FuncSyntax(GetTokens(startIndex), prototype, body);
} }
private StructSyntax ParseStruct(int startIndex, bool exported) private StructSyntax ParseStruct(int startIndex, bool exported, bool packed)
{ {
var name = ExpectIdentifier(); var name = ExpectIdentifier();
@@ -160,7 +146,7 @@ public sealed class Parser
{ {
var memberStartIndex = _tokenIndex; var memberStartIndex = _tokenIndex;
var fieldName = ExpectIdentifier().Value; var fieldName = ExpectIdentifier();
ExpectSymbol(Symbol.Colon); ExpectSymbol(Symbol.Colon);
var fieldType = ParseType(); var fieldType = ParseType();
@@ -174,7 +160,7 @@ public sealed class Parser
fields.Add(new StructFieldSyntax(GetTokens(memberStartIndex), fieldName, fieldType, fieldValue)); fields.Add(new StructFieldSyntax(GetTokens(memberStartIndex), fieldName, fieldType, fieldValue));
} }
return new StructSyntax(GetTokens(startIndex), name.Value, exported, fields); return new StructSyntax(GetTokens(startIndex), name, exported, packed, fields);
} }
private EnumSyntax ParseEnum(int startIndex, bool exported) private EnumSyntax ParseEnum(int startIndex, bool exported)
@@ -192,64 +178,65 @@ public sealed class Parser
ExpectSymbol(Symbol.OpenBrace); ExpectSymbol(Symbol.OpenBrace);
long value = -1;
while (!TryExpectSymbol(Symbol.CloseBrace)) while (!TryExpectSymbol(Symbol.CloseBrace))
{ {
var memberStartIndex = _tokenIndex; var memberStartIndex = _tokenIndex;
var fieldName = ExpectIdentifier().Value; var fieldName = ExpectIdentifier();
long fieldValue; IntLiteralToken? value = null;
if (TryExpectSymbol(Symbol.Assign)) if (TryExpectSymbol(Symbol.Assign))
{ {
if (!TryExpectIntLiteral(out var intLiteralToken)) if (!TryExpectIntLiteral(out var intLiteralToken))
{ {
throw new ParseException(Diagnostic throw new CompileException(Diagnostic
.Error("Value of enum field must be an integer literal") .Error("Value of enum field must be an integer literal")
.At(CurrentToken) .At(CurrentToken, _tokens)
.Build()); .Build());
} }
fieldValue = Convert.ToInt64(intLiteralToken.Value, intLiteralToken.Base); value = intLiteralToken;
value = fieldValue;
}
else
{
fieldValue = value + 1;
value = fieldValue;
} }
fields.Add(new EnumFieldSyntax(GetTokens(memberStartIndex), fieldName, fieldValue)); fields.Add(new EnumFieldSyntax(GetTokens(memberStartIndex), fieldName, value));
} }
return new EnumSyntax(GetTokens(startIndex), name.Value, exported, type, fields); return new EnumSyntax(GetTokens(startIndex), name, exported, type, fields);
} }
private StatementSyntax ParseStatement() private StatementSyntax ParseStatement()
{ {
var startIndex = _tokenIndex; var startIndex = _tokenIndex;
if (TryExpectSymbol(out var symbol)) if (CurrentToken is SymbolToken symbolToken)
{ {
switch (symbol) switch (symbolToken.Symbol)
{ {
case Symbol.OpenBrace: case Symbol.OpenBrace:
Next();
return ParseBlock(startIndex); return ParseBlock(startIndex);
case Symbol.Return: case Symbol.Return:
Next();
return ParseReturn(startIndex); return ParseReturn(startIndex);
case Symbol.If: case Symbol.If:
Next();
return ParseIf(startIndex); return ParseIf(startIndex);
case Symbol.While: case Symbol.While:
Next();
return ParseWhile(startIndex); return ParseWhile(startIndex);
case Symbol.For: case Symbol.For:
Next();
return ParseFor(startIndex); return ParseFor(startIndex);
case Symbol.Let: case Symbol.Let:
Next();
return ParseVariableDeclaration(startIndex); return ParseVariableDeclaration(startIndex);
case Symbol.Defer: case Symbol.Defer:
Next();
return ParseDefer(startIndex); return ParseDefer(startIndex);
case Symbol.Break: case Symbol.Break:
Next();
return new BreakSyntax(GetTokens(startIndex)); return new BreakSyntax(GetTokens(startIndex));
case Symbol.Continue: case Symbol.Continue:
Next();
return new ContinueSyntax(GetTokens(startIndex)); return new ContinueSyntax(GetTokens(startIndex));
} }
} }
@@ -267,7 +254,7 @@ public sealed class Parser
private VariableDeclarationSyntax ParseVariableDeclaration(int startIndex) private VariableDeclarationSyntax ParseVariableDeclaration(int startIndex)
{ {
var name = ExpectIdentifier().Value; var name = ExpectIdentifier();
TypeSyntax? explicitType = null; TypeSyntax? explicitType = null;
if (TryExpectSymbol(Symbol.Colon)) if (TryExpectSymbol(Symbol.Colon))
@@ -334,12 +321,12 @@ public sealed class Parser
private ForSyntax ParseFor(int startIndex) private ForSyntax ParseFor(int startIndex)
{ {
var itemName = ExpectIdentifier().Value; var itemName = ExpectIdentifier();
string? indexName = null; IdentifierToken? indexName = null;
if (TryExpectSymbol(Symbol.Comma)) if (TryExpectSymbol(Symbol.Comma))
{ {
indexName = ExpectIdentifier().Value; indexName = ExpectIdentifier();
} }
ExpectSymbol(Symbol.In); ExpectSymbol(Symbol.In);
@@ -354,7 +341,7 @@ public sealed class Parser
var startIndex = _tokenIndex; var startIndex = _tokenIndex;
var left = ParsePrimaryExpression(); var left = ParsePrimaryExpression();
while (CurrentToken is SymbolToken symbolToken && TryGetBinaryOperator(symbolToken.Symbol, out var op) && GetBinaryOperatorPrecedence(op.Value) >= precedence) while (CurrentToken is SymbolToken symbolToken && HasLeadingWhitespace(symbolToken) && HasTrailingWhitespace(symbolToken) && TryGetBinaryOperator(symbolToken.Symbol, out var op) && GetBinaryOperatorPrecedence(op.Value) >= precedence)
{ {
Next(); Next();
var right = ParseExpression(GetBinaryOperatorPrecedence(op.Value) + 1); var right = ParseExpression(GetBinaryOperatorPrecedence(op.Value) + 1);
@@ -452,7 +439,7 @@ public sealed class Parser
case Symbol.Pipe: case Symbol.Pipe:
binaryExpressionOperator = BinaryOperatorSyntax.BitwiseOr; binaryExpressionOperator = BinaryOperatorSyntax.BitwiseOr;
return true; return true;
case Symbol.Caret: case Symbol.Tilde:
binaryExpressionOperator = BinaryOperatorSyntax.BitwiseXor; binaryExpressionOperator = BinaryOperatorSyntax.BitwiseXor;
return true; return true;
default: default:
@@ -467,91 +454,47 @@ public sealed class Parser
var token = ExpectToken(); var token = ExpectToken();
var expr = token switch var expr = token switch
{ {
BoolLiteralToken boolLiteral => new BoolLiteralSyntax(GetTokens(startIndex), boolLiteral.Value), BoolLiteralToken boolLiteral => new BoolLiteralSyntax(GetTokens(startIndex), boolLiteral),
StringLiteralToken stringLiteral => new StringLiteralSyntax(GetTokens(startIndex), stringLiteral.Value), StringLiteralToken stringLiteral => new StringLiteralSyntax(GetTokens(startIndex), stringLiteral),
FloatLiteralToken floatLiteral => new FloatLiteralSyntax(GetTokens(startIndex), floatLiteral.Value), FloatLiteralToken floatLiteral => new FloatLiteralSyntax(GetTokens(startIndex), floatLiteral),
IntLiteralToken intLiteral => new IntLiteralSyntax(GetTokens(startIndex), intLiteral.Value, intLiteral.Base), IntLiteralToken intLiteral => new IntLiteralSyntax(GetTokens(startIndex), intLiteral),
IdentifierToken identifier => ParseIdentifier(startIndex, identifier), IdentifierToken identifier => ParseIdentifier(startIndex, identifier),
SymbolToken symbolToken => symbolToken.Symbol switch SymbolToken symbolToken => symbolToken.Symbol switch
{ {
Symbol.Caret => ParseAddressOf(startIndex),
Symbol.OpenParen => ParseParenthesizedExpression(), Symbol.OpenParen => ParseParenthesizedExpression(),
Symbol.Minus => new UnaryExpressionSyntax(GetTokens(startIndex), UnaryOperatorSyntax.Negate, ParsePrimaryExpression()), Symbol.Minus => ParseUnaryNegate(startIndex),
Symbol.Bang => new UnaryExpressionSyntax(GetTokens(startIndex), UnaryOperatorSyntax.Invert, ParsePrimaryExpression()), Symbol.Bang => ParseUnaryInvert(startIndex),
Symbol.OpenBracket => ParseArrayInitializer(startIndex), Symbol.OpenBracket => ParseArrayInitializer(startIndex),
Symbol.OpenBrace => new StructInitializerSyntax(GetTokens(startIndex), null, ParseStructInitializerBody()), Symbol.OpenBrace => ParseUnnamedStructInitializer(startIndex),
Symbol.Struct => ParseStructInitializer(startIndex), Symbol.Struct => ParseStructInitializer(startIndex),
Symbol.At => ParseBuiltinFunction(startIndex), Symbol.At => ParseBuiltinFunction(startIndex),
_ => throw new ParseException(Diagnostic _ => throw new CompileException(Diagnostic
.Error($"Unexpected symbol '{symbolToken.Symbol}' in expression") .Error($"Unexpected symbol '{symbolToken.Symbol}' in expression")
.WithHelp("Expected '(', '-', '!', '[' or '{'") .WithHelp("Expected '(', '-', '!', '[' or '{'")
.At(symbolToken) .At(symbolToken, _tokens)
.Build()) .Build())
}, },
_ => throw new ParseException(Diagnostic _ => throw new CompileException(Diagnostic
.Error($"Unexpected token '{token.GetType().Name}' in expression") .Error($"Unexpected token '{token.GetType().Name}' in expression")
.WithHelp("Expected literal, identifier, or parenthesized expression") .WithHelp("Expected literal, identifier, or parenthesized expression")
.At(token) .At(token, _tokens)
.Build()) .Build())
}; };
return ParsePostfixOperators(expr); return ParsePostfixOperators(expr);
} }
private ExpressionSyntax ParseBuiltinFunction(int startIndex)
{
var name = ExpectIdentifier();
ExpectSymbol(Symbol.OpenParen);
switch (name.Value)
{
case "size":
{
var type = ParseType();
ExpectSymbol(Symbol.CloseParen);
return new SizeSyntax(GetTokens(startIndex), type);
}
case "cast":
{
var expression = ParseExpression();
ExpectSymbol(Symbol.CloseParen);
return new CastSyntax(GetTokens(startIndex), expression);
}
default:
{
throw new ParseException(Diagnostic.Error($"Unknown builtin {name.Value}").At(name).Build());
}
}
}
private ExpressionSyntax ParseIdentifier(int startIndex, IdentifierToken identifier)
{
if (TryExpectSymbol(Symbol.DoubleColon))
{
var name = ExpectIdentifier();
return new ModuleIdentifierSyntax(GetTokens(startIndex), identifier.Value, name.Value);
}
return new LocalIdentifierSyntax(GetTokens(startIndex), identifier.Value);
}
private ExpressionSyntax ParseParenthesizedExpression()
{
var expression = ParseExpression();
ExpectSymbol(Symbol.CloseParen);
return expression;
}
private ExpressionSyntax ParsePostfixOperators(ExpressionSyntax expr) private ExpressionSyntax ParsePostfixOperators(ExpressionSyntax expr)
{ {
if (CurrentToken == null || HasLeadingWhitespace(CurrentToken))
{
return expr;
}
var startIndex = _tokenIndex; var startIndex = _tokenIndex;
while (HasToken) while (HasToken)
{ {
if (TryExpectSymbol(Symbol.Ampersand))
{
expr = new AddressOfSyntax(GetTokens(startIndex), expr);
continue;
}
if (TryExpectSymbol(Symbol.Caret)) if (TryExpectSymbol(Symbol.Caret))
{ {
expr = new DereferenceSyntax(GetTokens(startIndex), expr); expr = new DereferenceSyntax(GetTokens(startIndex), expr);
@@ -560,7 +503,7 @@ public sealed class Parser
if (TryExpectSymbol(Symbol.Period)) if (TryExpectSymbol(Symbol.Period))
{ {
var member = ExpectIdentifier().Value; var member = ExpectIdentifier();
expr = new MemberAccessSyntax(GetTokens(startIndex), expr, member); expr = new MemberAccessSyntax(GetTokens(startIndex), expr, member);
continue; continue;
} }
@@ -597,6 +540,68 @@ public sealed class Parser
return expr; return expr;
} }
private ExpressionSyntax ParseParenthesizedExpression()
{
var expression = ParseExpression();
ExpectSymbol(Symbol.CloseParen);
return expression;
}
private AddressOfSyntax ParseAddressOf(int startIndex)
{
var expression = ParsePrimaryExpression();
return new AddressOfSyntax(GetTokens(startIndex), expression);
}
private UnaryExpressionSyntax ParseUnaryInvert(int startIndex)
{
var expression = ParsePrimaryExpression();
return new UnaryExpressionSyntax(GetTokens(startIndex), UnaryOperatorSyntax.Invert, expression);
}
private UnaryExpressionSyntax ParseUnaryNegate(int startIndex)
{
var expression = ParsePrimaryExpression();
return new UnaryExpressionSyntax(GetTokens(startIndex), UnaryOperatorSyntax.Negate, expression);
}
private ExpressionSyntax ParseBuiltinFunction(int startIndex)
{
var name = ExpectIdentifier();
ExpectSymbol(Symbol.OpenParen);
switch (name.Value)
{
case "size":
{
var type = ParseType();
ExpectSymbol(Symbol.CloseParen);
return new SizeSyntax(GetTokens(startIndex), type);
}
case "cast":
{
var expression = ParseExpression();
ExpectSymbol(Symbol.CloseParen);
return new CastSyntax(GetTokens(startIndex), expression);
}
default:
{
throw new CompileException(Diagnostic.Error($"Unknown builtin {name.Value}").At(name).Build());
}
}
}
private ExpressionSyntax ParseIdentifier(int startIndex, IdentifierToken identifier)
{
if (TryExpectSymbol(Symbol.DoubleColon))
{
var name = ExpectIdentifier();
return new ModuleIdentifierSyntax(GetTokens(startIndex), identifier, name);
}
return new LocalIdentifierSyntax(GetTokens(startIndex), identifier);
}
private ExpressionSyntax ParseArrayInitializer(int startIndex) private ExpressionSyntax ParseArrayInitializer(int startIndex)
{ {
var values = new List<ExpressionSyntax>(); var values = new List<ExpressionSyntax>();
@@ -627,12 +632,18 @@ public sealed class Parser
return new StructInitializerSyntax(GetTokens(startIndex), type, initializers); return new StructInitializerSyntax(GetTokens(startIndex), type, initializers);
} }
private Dictionary<string, ExpressionSyntax> ParseStructInitializerBody() private StructInitializerSyntax ParseUnnamedStructInitializer(int startIndex)
{ {
Dictionary<string, ExpressionSyntax> initializers = []; var body = ParseStructInitializerBody();
return new StructInitializerSyntax(GetTokens(startIndex), null, body);
}
private Dictionary<IdentifierToken, ExpressionSyntax> ParseStructInitializerBody()
{
Dictionary<IdentifierToken, ExpressionSyntax> initializers = [];
while (!TryExpectSymbol(Symbol.CloseBrace)) while (!TryExpectSymbol(Symbol.CloseBrace))
{ {
var name = ExpectIdentifier().Value; var name = ExpectIdentifier();
ExpectSymbol(Symbol.Assign); ExpectSymbol(Symbol.Assign);
var value = ParseExpression(); var value = ParseExpression();
initializers.Add(name, value); initializers.Add(name, value);
@@ -658,7 +669,7 @@ public sealed class Parser
{ {
statements.Add(ParseStatement()); statements.Add(ParseStatement());
} }
catch (ParseException ex) catch (CompileException ex)
{ {
Diagnostics.Add(ex.Diagnostic); Diagnostics.Add(ex.Diagnostic);
if (HasToken) if (HasToken)
@@ -680,42 +691,42 @@ public sealed class Parser
var startIndex = _tokenIndex; var startIndex = _tokenIndex;
if (TryExpectIdentifier(out var name)) if (TryExpectIdentifier(out var name))
{ {
if (name.Value[0] == 'u' && int.TryParse(name.Value[1..], out var size)) if (name.Value[0] == 'u' && ulong.TryParse(name.Value[1..], out var size))
{ {
if (size is not 8 and not 16 and not 32 and not 64) if (size is not 8 and not 16 and not 32 and not 64)
{ {
throw new ParseException(Diagnostic throw new CompileException(Diagnostic
.Error("Arbitrary uint size is not supported") .Error("Arbitrary uint size is not supported")
.WithHelp("Use u8, u16, u32 or u64") .WithHelp("Use u8, u16, u32 or u64")
.At(name) .At(name, _tokens)
.Build()); .Build());
} }
return new IntTypeSyntax(GetTokens(startIndex), false, size); return new IntTypeSyntax(GetTokens(startIndex), false, size);
} }
if (name.Value[0] == 'i' && int.TryParse(name.Value[1..], out size)) if (name.Value[0] == 'i' && ulong.TryParse(name.Value[1..], out size))
{ {
if (size is not 8 and not 16 and not 32 and not 64) if (size is not 8 and not 16 and not 32 and not 64)
{ {
throw new ParseException(Diagnostic throw new CompileException(Diagnostic
.Error("Arbitrary int size is not supported") .Error("Arbitrary int size is not supported")
.WithHelp("Use i8, i16, i32 or i64") .WithHelp("Use i8, i16, i32 or i64")
.At(name) .At(name, _tokens)
.Build()); .Build());
} }
return new IntTypeSyntax(GetTokens(startIndex), true, size); return new IntTypeSyntax(GetTokens(startIndex), true, size);
} }
if (name.Value[0] == 'f' && int.TryParse(name.Value[1..], out size)) if (name.Value[0] == 'f' && ulong.TryParse(name.Value[1..], out size))
{ {
if (size is not 32 and not 64) if (size is not 32 and not 64)
{ {
throw new ParseException(Diagnostic throw new CompileException(Diagnostic
.Error("Arbitrary float size is not supported") .Error("Arbitrary float size is not supported")
.WithHelp("Use f32 or f64") .WithHelp("Use f32 or f64")
.At(name) .At(name, _tokens)
.Build()); .Build());
} }
@@ -732,16 +743,16 @@ public sealed class Parser
return new BoolTypeSyntax(GetTokens(startIndex)); return new BoolTypeSyntax(GetTokens(startIndex));
default: default:
{ {
string? module = null; IdentifierToken? module = null;
if (TryExpectSymbol(Symbol.DoubleColon)) if (TryExpectSymbol(Symbol.DoubleColon))
{ {
var customTypeName = ExpectIdentifier(); var customTypeName = ExpectIdentifier();
module = name.Value; module = name;
name = customTypeName; name = customTypeName;
} }
return new CustomTypeSyntax(GetTokens(startIndex), module, name.Value); return new CustomTypeSyntax(GetTokens(startIndex), module, name);
} }
} }
} }
@@ -780,7 +791,7 @@ public sealed class Parser
{ {
ExpectSymbol(Symbol.CloseBracket); ExpectSymbol(Symbol.CloseBracket);
var baseType = ParseType(); var baseType = ParseType();
return new ConstArrayTypeSyntax(GetTokens(startIndex), baseType, Convert.ToInt64(intLiteral.Value, intLiteral.Base)); return new ConstArrayTypeSyntax(GetTokens(startIndex), baseType, intLiteral.AsU64);
} }
else if (TryExpectSymbol(Symbol.QuestionMark)) else if (TryExpectSymbol(Symbol.QuestionMark))
{ {
@@ -796,10 +807,10 @@ public sealed class Parser
} }
} }
throw new ParseException(Diagnostic throw new CompileException(Diagnostic
.Error("Invalid type syntax") .Error("Invalid type syntax")
.WithHelp("Expected type name, '^' for pointer, or '[]' for array") .WithHelp("Expected type name, '^' for pointer, or '[]' for array")
.At(CurrentToken) .At(CurrentToken, _tokens)
.Build()); .Build());
} }
@@ -807,10 +818,10 @@ public sealed class Parser
{ {
if (!HasToken) if (!HasToken)
{ {
throw new ParseException(Diagnostic throw new CompileException(Diagnostic
.Error("Unexpected end of file") .Error("Unexpected end of file")
.WithHelp("Expected more tokens to complete the syntax") .WithHelp("Expected more tokens to complete the syntax")
.At(_tokens[^1]) .At(_tokens[^1], _tokens)
.Build()); .Build());
} }
@@ -824,10 +835,10 @@ public sealed class Parser
var token = ExpectToken(); var token = ExpectToken();
if (token is not SymbolToken symbol) if (token is not SymbolToken symbol)
{ {
throw new ParseException(Diagnostic throw new CompileException(Diagnostic
.Error($"Expected symbol, but found {token.GetType().Name}") .Error($"Expected symbol, but found {token.GetType().Name}")
.WithHelp("This position requires a symbol like '(', ')', '{', '}', etc.") .WithHelp("This position requires a symbol like '(', ')', '{', '}', etc.")
.At(token) .At(token, _tokens)
.Build()); .Build());
} }
@@ -839,10 +850,10 @@ public sealed class Parser
var token = ExpectSymbol(); var token = ExpectSymbol();
if (token.Symbol != expectedSymbol) if (token.Symbol != expectedSymbol)
{ {
throw new ParseException(Diagnostic throw new CompileException(Diagnostic
.Error($"Expected '{expectedSymbol}', but found '{token.Symbol}'") .Error($"Expected '{expectedSymbol}', but found '{token.Symbol}'")
.WithHelp($"Insert '{expectedSymbol}' here") .WithHelp($"Insert '{expectedSymbol}' here")
.At(token) .At(token, _tokens)
.Build()); .Build());
} }
} }
@@ -889,10 +900,10 @@ public sealed class Parser
var token = ExpectToken(); var token = ExpectToken();
if (token is not IdentifierToken identifier) if (token is not IdentifierToken identifier)
{ {
throw new ParseException(Diagnostic throw new CompileException(Diagnostic
.Error($"Expected identifier, but found {token.GetType().Name}") .Error($"Expected identifier, but found {token.GetType().Name}")
.WithHelp("Provide a valid identifier name here") .WithHelp("Provide a valid identifier name here")
.At(token) .At(token, _tokens)
.Build()); .Build());
} }
@@ -917,10 +928,10 @@ public sealed class Parser
var token = ExpectToken(); var token = ExpectToken();
if (token is not StringLiteralToken identifier) if (token is not StringLiteralToken identifier)
{ {
throw new ParseException(Diagnostic throw new CompileException(Diagnostic
.Error($"Expected string literal, but found {token.GetType().Name}") .Error($"Expected string literal, but found {token.GetType().Name}")
.WithHelp("Provide a valid string literal") .WithHelp("Provide a valid string literal")
.At(token) .At(token, _tokens)
.Build()); .Build());
} }
@@ -930,6 +941,10 @@ public sealed class Parser
private void Next() private void Next()
{ {
_tokenIndex++; _tokenIndex++;
while (_tokenIndex < _tokens.Count && _tokens[_tokenIndex] is WhitespaceToken or CommentToken)
{
_tokenIndex++;
}
} }
private List<Token> GetTokens(int tokenStartIndex) private List<Token> GetTokens(int tokenStartIndex)
@@ -938,14 +953,4 @@ public sealed class Parser
} }
} }
public record SyntaxTree(List<DefinitionSyntax> Definitions, string ModuleName, List<string> Imports); public record SyntaxTree(List<TopLevelSyntaxNode> TopLevelSyntaxNodes, List<Token> Tokens);
public class ParseException : Exception
{
public Diagnostic Diagnostic { get; }
public ParseException(Diagnostic diagnostic) : base(diagnostic.Message)
{
Diagnostic = diagnostic;
}
}

View File

@@ -4,21 +4,25 @@ public abstract record SyntaxNode(List<Token> Tokens);
#region Definitions #region Definitions
public abstract record DefinitionSyntax(List<Token> Tokens, string Name, bool Exported) : SyntaxNode(Tokens); public record TopLevelSyntaxNode(List<Token> Tokens) : SyntaxNode(Tokens);
public record FuncParameterSyntax(List<Token> Tokens, string Name, TypeSyntax Type) : SyntaxNode(Tokens); public record ModuleSyntax(List<Token> Tokens, IdentifierToken NameToken) : TopLevelSyntaxNode(Tokens);
public record FuncPrototypeSyntax(List<Token> Tokens, string Name, bool Exported, string? ExternSymbol, List<FuncParameterSyntax> Parameters, TypeSyntax ReturnType) : SyntaxNode(Tokens); public abstract record DefinitionSyntax(List<Token> Tokens, IdentifierToken NameToken, bool Exported) : TopLevelSyntaxNode(Tokens);
public record FuncSyntax(List<Token> Tokens, FuncPrototypeSyntax Prototype, BlockSyntax? Body) : DefinitionSyntax(Tokens, Prototype.Name, Prototype.Exported); public record FuncParameterSyntax(List<Token> Tokens, IdentifierToken NameToken, TypeSyntax Type) : SyntaxNode(Tokens);
public record StructFieldSyntax(List<Token> Tokens, string Name, TypeSyntax Type, ExpressionSyntax? Value) : SyntaxNode(Tokens); public record FuncPrototypeSyntax(List<Token> Tokens, IdentifierToken NameToken, bool Exported, StringLiteralToken? ExternSymbolToken, List<FuncParameterSyntax> Parameters, TypeSyntax ReturnType) : SyntaxNode(Tokens);
public record StructSyntax(List<Token> Tokens, string Name, bool Exported, List<StructFieldSyntax> Fields) : DefinitionSyntax(Tokens, Name, Exported); public record FuncSyntax(List<Token> Tokens, FuncPrototypeSyntax Prototype, BlockSyntax? Body) : DefinitionSyntax(Tokens, Prototype.NameToken, Prototype.Exported);
public record EnumFieldSyntax(List<Token> Tokens, string Name, long Value) : SyntaxNode(Tokens); public record StructFieldSyntax(List<Token> Tokens, IdentifierToken NameToken, TypeSyntax Type, ExpressionSyntax? Value) : SyntaxNode(Tokens);
public record EnumSyntax(List<Token> Tokens, string Name, bool Exported, TypeSyntax? Type, List<EnumFieldSyntax> Fields) : DefinitionSyntax(Tokens, Name, Exported); public record StructSyntax(List<Token> Tokens, IdentifierToken NameToken, bool Exported, bool Packed, List<StructFieldSyntax> Fields) : DefinitionSyntax(Tokens, NameToken, Exported);
public record EnumFieldSyntax(List<Token> Tokens, IdentifierToken NameToken, IntLiteralToken? ValueToken) : SyntaxNode(Tokens);
public record EnumSyntax(List<Token> Tokens, IdentifierToken NameToken, bool Exported, TypeSyntax? Type, List<EnumFieldSyntax> Fields) : DefinitionSyntax(Tokens, NameToken, Exported);
public enum UnaryOperatorSyntax public enum UnaryOperatorSyntax
{ {
@@ -64,7 +68,7 @@ public record AssignmentSyntax(List<Token> Tokens, ExpressionSyntax Target, Expr
public record IfSyntax(List<Token> Tokens, ExpressionSyntax Condition, BlockSyntax Body, Variant<IfSyntax, BlockSyntax>? Else) : StatementSyntax(Tokens); public record IfSyntax(List<Token> Tokens, ExpressionSyntax Condition, BlockSyntax Body, Variant<IfSyntax, BlockSyntax>? Else) : StatementSyntax(Tokens);
public record VariableDeclarationSyntax(List<Token> Tokens, string Name, TypeSyntax? ExplicitType, ExpressionSyntax? Assignment) : StatementSyntax(Tokens); public record VariableDeclarationSyntax(List<Token> Tokens, IdentifierToken NameToken, TypeSyntax? ExplicitType, ExpressionSyntax? Assignment) : StatementSyntax(Tokens);
public record ContinueSyntax(List<Token> Tokens) : StatementSyntax(Tokens); public record ContinueSyntax(List<Token> Tokens) : StatementSyntax(Tokens);
@@ -74,7 +78,7 @@ public record DeferSyntax(List<Token> Tokens, StatementSyntax Statement) : State
public record WhileSyntax(List<Token> Tokens, ExpressionSyntax Condition, BlockSyntax Body) : StatementSyntax(Tokens); public record WhileSyntax(List<Token> Tokens, ExpressionSyntax Condition, BlockSyntax Body) : StatementSyntax(Tokens);
public record ForSyntax(List<Token> Tokens, string ElementName, string? IndexName, ExpressionSyntax Target, BlockSyntax Body) : StatementSyntax(Tokens); public record ForSyntax(List<Token> Tokens, IdentifierToken ElementNameToken, IdentifierToken? IndexNameToken, ExpressionSyntax Target, BlockSyntax Body) : StatementSyntax(Tokens);
#endregion #endregion
@@ -88,9 +92,9 @@ public record UnaryExpressionSyntax(List<Token> Tokens, UnaryOperatorSyntax Oper
public record FuncCallSyntax(List<Token> Tokens, ExpressionSyntax Expression, List<ExpressionSyntax> Parameters) : ExpressionSyntax(Tokens); public record FuncCallSyntax(List<Token> Tokens, ExpressionSyntax Expression, List<ExpressionSyntax> Parameters) : ExpressionSyntax(Tokens);
public record LocalIdentifierSyntax(List<Token> Tokens, string Name) : ExpressionSyntax(Tokens); public record LocalIdentifierSyntax(List<Token> Tokens, IdentifierToken NameToken) : ExpressionSyntax(Tokens);
public record ModuleIdentifierSyntax(List<Token> Tokens, string Module, string Name) : ExpressionSyntax(Tokens); public record ModuleIdentifierSyntax(List<Token> Tokens, IdentifierToken ModuleToken, IdentifierToken NameToken) : ExpressionSyntax(Tokens);
public record ArrayInitializerSyntax(List<Token> Tokens, List<ExpressionSyntax> Values) : ExpressionSyntax(Tokens); public record ArrayInitializerSyntax(List<Token> Tokens, List<ExpressionSyntax> Values) : ExpressionSyntax(Tokens);
@@ -98,17 +102,17 @@ public record ArrayIndexAccessSyntax(List<Token> Tokens, ExpressionSyntax Target
public record AddressOfSyntax(List<Token> Tokens, ExpressionSyntax Target) : ExpressionSyntax(Tokens); public record AddressOfSyntax(List<Token> Tokens, ExpressionSyntax Target) : ExpressionSyntax(Tokens);
public record IntLiteralSyntax(List<Token> Tokens, string Value, int Base) : ExpressionSyntax(Tokens); public record IntLiteralSyntax(List<Token> Tokens, IntLiteralToken Token) : ExpressionSyntax(Tokens);
public record StringLiteralSyntax(List<Token> Tokens, string Value) : ExpressionSyntax(Tokens); public record StringLiteralSyntax(List<Token> Tokens, StringLiteralToken Token) : ExpressionSyntax(Tokens);
public record BoolLiteralSyntax(List<Token> Tokens, bool Value) : ExpressionSyntax(Tokens); public record BoolLiteralSyntax(List<Token> Tokens, BoolLiteralToken Token) : ExpressionSyntax(Tokens);
public record FloatLiteralSyntax(List<Token> Tokens, string Value) : ExpressionSyntax(Tokens); public record FloatLiteralSyntax(List<Token> Tokens, FloatLiteralToken Token) : ExpressionSyntax(Tokens);
public record MemberAccessSyntax(List<Token> Tokens, ExpressionSyntax Target, string Member) : ExpressionSyntax(Tokens); public record MemberAccessSyntax(List<Token> Tokens, ExpressionSyntax Target, IdentifierToken MemberToken) : ExpressionSyntax(Tokens);
public record StructInitializerSyntax(List<Token> Tokens, TypeSyntax? StructType, Dictionary<string, ExpressionSyntax> Initializers) : ExpressionSyntax(Tokens); public record StructInitializerSyntax(List<Token> Tokens, TypeSyntax? StructType, Dictionary<IdentifierToken, ExpressionSyntax> Initializers) : ExpressionSyntax(Tokens);
public record DereferenceSyntax(List<Token> Tokens, ExpressionSyntax Target) : ExpressionSyntax(Tokens); public record DereferenceSyntax(List<Token> Tokens, ExpressionSyntax Target) : ExpressionSyntax(Tokens);
@@ -128,9 +132,9 @@ public record PointerTypeSyntax(List<Token> Tokens, TypeSyntax BaseType) : TypeS
public record VoidTypeSyntax(List<Token> Tokens) : TypeSyntax(Tokens); public record VoidTypeSyntax(List<Token> Tokens) : TypeSyntax(Tokens);
public record IntTypeSyntax(List<Token> Tokens, bool Signed, int Width) : TypeSyntax(Tokens); public record IntTypeSyntax(List<Token> Tokens, bool Signed, ulong Width) : TypeSyntax(Tokens);
public record FloatTypeSyntax(List<Token> Tokens, int Width) : TypeSyntax(Tokens); public record FloatTypeSyntax(List<Token> Tokens, ulong Width) : TypeSyntax(Tokens);
public record BoolTypeSyntax(List<Token> Tokens) : TypeSyntax(Tokens); public record BoolTypeSyntax(List<Token> Tokens) : TypeSyntax(Tokens);
@@ -140,8 +144,8 @@ public record SliceTypeSyntax(List<Token> Tokens, TypeSyntax BaseType) : TypeSyn
public record ArrayTypeSyntax(List<Token> Tokens, TypeSyntax BaseType) : TypeSyntax(Tokens); public record ArrayTypeSyntax(List<Token> Tokens, TypeSyntax BaseType) : TypeSyntax(Tokens);
public record ConstArrayTypeSyntax(List<Token> Tokens, TypeSyntax BaseType, long Size) : TypeSyntax(Tokens); public record ConstArrayTypeSyntax(List<Token> Tokens, TypeSyntax BaseType, ulong Size) : TypeSyntax(Tokens);
public record CustomTypeSyntax(List<Token> Tokens, string? Module, string Name) : TypeSyntax(Tokens); public record CustomTypeSyntax(List<Token> Tokens, IdentifierToken? ModuleToken, IdentifierToken NameToken) : TypeSyntax(Tokens);
#endregion #endregion

View File

@@ -2,8 +2,105 @@
namespace NubLang.Syntax; namespace NubLang.Syntax;
public abstract class Token(SourceSpan span)
{
public SourceSpan Span { get; } = span;
}
public class WhitespaceToken(SourceSpan span) : Token(span);
public class CommentToken(SourceSpan span, string comment) : Token(span)
{
public string Comment { get; } = comment;
public override string ToString()
{
return "// " + Comment;
}
}
public class IdentifierToken(SourceSpan span, string value) : Token(span)
{
public string Value { get; } = value;
public override string ToString()
{
return Value;
}
}
public class IntLiteralToken(SourceSpan span, string value, int @base) : Token(span)
{
public string Value { get; } = value;
public int Base { get; } = @base;
private string GetNumericValue()
{
return Base switch
{
2 when Value.StartsWith("0b", StringComparison.OrdinalIgnoreCase) => Value[2..],
8 when Value.StartsWith("0o", StringComparison.OrdinalIgnoreCase) => Value[2..],
16 when Value.StartsWith("0x", StringComparison.OrdinalIgnoreCase) => Value[2..],
_ => Value
};
}
public ulong AsU64 => Convert.ToUInt64(GetNumericValue(), Base);
public long AsI64 => Convert.ToInt64(GetNumericValue(), Base);
public uint AsU32 => Convert.ToUInt32(GetNumericValue(), Base);
public int AsI32 => Convert.ToInt32(GetNumericValue(), Base);
public ushort AsU16 => Convert.ToUInt16(GetNumericValue(), Base);
public short AsI16 => Convert.ToInt16(GetNumericValue(), Base);
public byte AsU8 => Convert.ToByte(GetNumericValue(), Base);
public sbyte AsI8 => Convert.ToSByte(GetNumericValue(), Base);
public float AsF32 => Convert.ToSingle(AsI32);
public double AsF64 => Convert.ToDouble(AsI64);
public override string ToString()
{
return Value;
}
}
public class StringLiteralToken(SourceSpan span, string value) : Token(span)
{
public string Value { get; } = value;
public override string ToString()
{
return $"\"{Value}\"";
}
}
public class BoolLiteralToken(SourceSpan span, bool value) : Token(span)
{
public bool Value { get; } = value;
public override string ToString()
{
return Value ? "true" : "false";
}
}
public class FloatLiteralToken(SourceSpan span, string value) : Token(span)
{
public string Value { get; } = value;
public float AsF32 => Convert.ToSingle(Value);
public double AsF64 => Convert.ToDouble(Value);
public override string ToString()
{
return Value;
}
}
public enum Symbol public enum Symbol
{ {
// Default
None,
// Control // Control
If, If,
Else, Else,
@@ -20,11 +117,11 @@ public enum Symbol
Func, Func,
Struct, Struct,
Enum, Enum,
Import,
Module, Module,
// Modifier // Modifier
Extern, Extern,
Packed,
Export, Export,
Colon, Colon,
@@ -50,6 +147,7 @@ public enum Symbol
Star, Star,
ForwardSlash, ForwardSlash,
Caret, Caret,
Tilde,
Ampersand, Ampersand,
Semi, Semi,
Percent, Percent,
@@ -62,16 +160,64 @@ public enum Symbol
QuestionMark, QuestionMark,
} }
public abstract record Token(SourceSpan Span); public class SymbolToken(SourceSpan span, Symbol symbol) : Token(span)
{
public Symbol Symbol { get; } = symbol;
public record IdentifierToken(SourceSpan Span, string Value) : Token(Span); public override string ToString()
{
public record IntLiteralToken(SourceSpan Span, string Value, int Base) : Token(Span); return Symbol switch
{
public record StringLiteralToken(SourceSpan Span, string Value) : Token(Span); Symbol.Func => "func",
Symbol.If => "if",
public record BoolLiteralToken(SourceSpan Span, bool Value) : Token(Span); Symbol.Else => "else",
Symbol.While => "while",
public record FloatLiteralToken(SourceSpan Span, string Value) : Token(Span); Symbol.For => "for",
Symbol.In => "in",
public record SymbolToken(SourceSpan Span, Symbol Symbol) : Token(Span); Symbol.Break => "break",
Symbol.Continue => "continue",
Symbol.Return => "return",
Symbol.Struct => "struct",
Symbol.Let => "let",
Symbol.Extern => "extern",
Symbol.Module => "module",
Symbol.Export => "export",
Symbol.Defer => "defer",
Symbol.Enum => "enum",
Symbol.Equal => "==",
Symbol.NotEqual => "!=",
Symbol.LessThanOrEqual => "<=",
Symbol.GreaterThanOrEqual => ">=",
Symbol.LeftShift => "<<",
Symbol.RightShift => ">>",
Symbol.And => "&&",
Symbol.Or => "||",
Symbol.DoubleColon => "::",
Symbol.Colon => ":",
Symbol.OpenParen => "(",
Symbol.CloseParen => ")",
Symbol.OpenBrace => "{",
Symbol.CloseBrace => "}",
Symbol.OpenBracket => "[",
Symbol.CloseBracket => "]",
Symbol.Comma => ",",
Symbol.Period => ".",
Symbol.Assign => "=",
Symbol.LessThan => "<",
Symbol.GreaterThan => ">",
Symbol.Plus => "+",
Symbol.Minus => "-",
Symbol.Star => "*",
Symbol.ForwardSlash => "/",
Symbol.Bang => "!",
Symbol.Caret => "^",
Symbol.Ampersand => "&",
Symbol.Semi => ";",
Symbol.Percent => "%",
Symbol.Pipe => "|",
Symbol.At => "@",
Symbol.QuestionMark => "?",
_ => "none",
};
}
}

View File

@@ -4,328 +4,406 @@ namespace NubLang.Syntax;
public sealed class Tokenizer public sealed class Tokenizer
{ {
private static readonly Dictionary<string, Symbol> Keywords = new() private string _fileName = null!;
{ private string _content = null!;
["func"] = Symbol.Func, private int _index;
["if"] = Symbol.If,
["else"] = Symbol.Else,
["while"] = Symbol.While,
["for"] = Symbol.For,
["in"] = Symbol.In,
["break"] = Symbol.Break,
["continue"] = Symbol.Continue,
["return"] = Symbol.Return,
["struct"] = Symbol.Struct,
["let"] = Symbol.Let,
["extern"] = Symbol.Extern,
["module"] = Symbol.Module,
["export"] = Symbol.Export,
["import"] = Symbol.Import,
["defer"] = Symbol.Defer,
["enum"] = Symbol.Enum,
};
private static readonly Dictionary<char[], Symbol> Symbols = new()
{
[['=', '=']] = Symbol.Equal,
[['!', '=']] = Symbol.NotEqual,
[['<', '=']] = Symbol.LessThanOrEqual,
[['>', '=']] = Symbol.GreaterThanOrEqual,
[['<', '<']] = Symbol.LeftShift,
[['>', '>']] = Symbol.RightShift,
[['&', '&']] = Symbol.And,
[['|', '|']] = Symbol.Or,
[[':', ':']] = Symbol.DoubleColon,
[[':']] = Symbol.Colon,
[['(']] = Symbol.OpenParen,
[[')']] = Symbol.CloseParen,
[['{']] = Symbol.OpenBrace,
[['}']] = Symbol.CloseBrace,
[['[']] = Symbol.OpenBracket,
[[']']] = Symbol.CloseBracket,
[[',']] = Symbol.Comma,
[['.']] = Symbol.Period,
[['=']] = Symbol.Assign,
[['<']] = Symbol.LessThan,
[['>']] = Symbol.GreaterThan,
[['+']] = Symbol.Plus,
[['-']] = Symbol.Minus,
[['*']] = Symbol.Star,
[['/']] = Symbol.ForwardSlash,
[['!']] = Symbol.Bang,
[['^']] = Symbol.Caret,
[['&']] = Symbol.Ampersand,
[[';']] = Symbol.Semi,
[['%']] = Symbol.Percent,
[['|']] = Symbol.Pipe,
[['@']] = Symbol.At,
[['?']] = Symbol.QuestionMark,
};
private static readonly (char[] Pattern, Symbol Symbol)[] OrderedSymbols = Symbols
.OrderByDescending(kvp => kvp.Key.Length)
.Select(kvp => (kvp.Key, kvp.Value))
.ToArray();
private readonly string _fileName;
private readonly string _content;
private int _index = 0;
private int _line = 1; private int _line = 1;
private int _column = 1; private int _column = 1;
public Tokenizer(string fileName, string content) public List<Diagnostic> Diagnostics { get; set; } = new(16);
public List<Token> Tokenize(string fileName, string content)
{ {
_fileName = fileName; _fileName = fileName;
_content = content; _content = content;
}
public List<Diagnostic> Diagnostics { get; } = []; Diagnostics = [];
public List<Token> Tokens { get; } = [];
public void Tokenize()
{
Diagnostics.Clear();
Tokens.Clear();
_index = 0; _index = 0;
_line = 1; _line = 1;
_column = 1; _column = 1;
while (Peek().HasValue) var tokens = new List<Token>();
while (_index < _content.Length)
{ {
try try
{ {
var current = Peek()!.Value; tokens.Add(ParseToken());
if (char.IsWhiteSpace(current))
{
if (current is '\n')
{
_line += 1;
// note(nub31): Next increments the column, so 0 is correct here
_column = 0;
}
Next();
continue;
}
if (current == '/' && Peek(1) == '/')
{
// note(nub31): Keep newline so next iteration increments the line counter
while (Peek() is not '\n')
{
Next();
}
continue;
}
Tokens.Add(ParseToken(current, _line, _column));
} }
catch (TokenizerException e) catch (CompileException e)
{ {
Diagnostics.Add(e.Diagnostic); Diagnostics.Add(e.Diagnostic);
Next(); Next();
} }
} }
return tokens;
} }
private Token ParseToken(char current, int lineStart, int columnStart) private Token ParseToken()
{ {
if (char.IsLetter(current) || current == '_') var indexStart = _index;
{ var lineStart = _line;
var buffer = string.Empty; var columnStart = _column;
while (Peek() != null && (char.IsLetterOrDigit(Peek()!.Value) || Peek() == '_')) if (char.IsWhiteSpace(_content[_index]))
{
while (_index < _content.Length && char.IsWhiteSpace(_content[_index]))
{ {
buffer += Peek();
Next(); Next();
} }
if (Keywords.TryGetValue(buffer, out var keywordSymbol)) return new WhitespaceToken(CreateSpan(indexStart, lineStart, columnStart));
{
return new SymbolToken(CreateSpan(lineStart, columnStart), keywordSymbol);
}
if (buffer is "true" or "false")
{
return new BoolLiteralToken(CreateSpan(lineStart, columnStart), Convert.ToBoolean(buffer));
}
return new IdentifierToken(CreateSpan(lineStart, columnStart), buffer);
} }
if (char.IsDigit(current)) if (_content[_index] == '/' && _index + 1 < _content.Length && _content[_index + 1] == '/')
{ {
var buffer = string.Empty; var startIndex = _index;
if (current == '0' && Peek(1) is 'x') Next(2);
while (_index < _content.Length && _content[_index] != '\n')
{ {
buffer += "0x";
Next(); Next();
Next(); }
while (Peek() != null && Uri.IsHexDigit(Peek()!.Value))
{
buffer += Peek()!.Value;
Next();
}
if (buffer.Length <= 2) return new CommentToken(CreateSpan(indexStart, lineStart, columnStart), _content.AsSpan(startIndex, _index - startIndex).ToString());
}
if (char.IsDigit(_content[_index]))
{
return ParseNumber(indexStart, lineStart, columnStart);
}
if (_content[_index] == '"')
{
return ParseString(indexStart, lineStart, columnStart);
}
// note(nub31): Look for keywords (longest first in case a keyword fits partially in a larger keyword)
for (var i = 8; i >= 1; i--)
{
if (TryMatchSymbol(i, indexStart, lineStart, columnStart, out var token))
{
return token;
}
}
if (char.IsLetter(_content[_index]) || _content[_index] == '_')
{
return ParseIdentifier(indexStart, lineStart, columnStart);
}
throw new CompileException(Diagnostic.Error($"Unknown token '{_content[_index]}'").Build());
}
private Token ParseNumber(int indexStart, int lineStart, int columnStart)
{
var start = _index;
var current = _content[_index];
// note(nub31): 0xFFFFFF
if (current == '0' && _index + 1 < _content.Length && _content[_index + 1] == 'x')
{
Next(2);
var digitStart = _index;
while (_index < _content.Length && Uri.IsHexDigit(_content[_index]))
{
Next();
}
if (_index == digitStart)
{
throw new CompileException(Diagnostic
.Error("Invalid hex literal, no digits found")
.At(CreateSpan(_index, _line, _column))
.Build());
}
return new IntLiteralToken(
CreateSpan(indexStart, lineStart, columnStart),
_content.Substring(start, _index - start),
16);
}
// note(nub31): 0b11001100
if (current == '0' && _index + 1 < _content.Length && _content[_index + 1] == 'b')
{
Next(2);
var digitStart = _index;
while (_index < _content.Length && (_content[_index] == '0' || _content[_index] == '1'))
{
Next();
}
if (_index == digitStart)
{
throw new CompileException(Diagnostic
.Error("Invalid binary literal, no digits found")
.At(CreateSpan(_index, _line, _column))
.Build());
}
return new IntLiteralToken(
CreateSpan(indexStart, lineStart, columnStart),
_content.Substring(start, _index - start),
2);
}
// note(nub31): 23/23.5
var isFloat = false;
while (_index < _content.Length)
{
var next = _content[_index];
if (next == '.')
{
if (isFloat)
{ {
throw new TokenizerException(Diagnostic throw new CompileException(Diagnostic
.Error("Invalid hex literal, no digits found") .Error("More than one period found in float literal")
.At(_fileName, _line, _column) .At(CreateSpan(_index, _line, _column))
.Build()); .Build());
} }
return new IntLiteralToken(CreateSpan(lineStart, columnStart), buffer, 16); isFloat = true;
}
if (current == '0' && Peek(1) is 'b')
{
buffer += "0b";
Next(); Next();
}
else if (char.IsDigit(next))
{
Next(); Next();
while (Peek() != null && (Peek() == '0' || Peek() == '1'))
{
buffer += Peek()!.Value;
Next();
}
if (buffer.Length <= 2)
{
throw new TokenizerException(Diagnostic
.Error("Invalid binary literal, no digits found")
.At(_fileName, _line, _column)
.Build());
}
return new IntLiteralToken(CreateSpan(lineStart, columnStart), buffer, 2);
}
var isFloat = false;
while (Peek() != null)
{
var next = Peek()!.Value;
if (next == '.')
{
if (isFloat)
{
throw new TokenizerException(Diagnostic
.Error("More than one period found in float literal")
.At(_fileName, _line, _column)
.Build());
}
isFloat = true;
buffer += next;
Next();
}
else if (char.IsDigit(next))
{
buffer += next;
Next();
}
else
{
break;
}
}
if (isFloat)
{
return new FloatLiteralToken(CreateSpan(lineStart, columnStart), buffer);
} }
else else
{ {
return new IntLiteralToken(CreateSpan(lineStart, columnStart), buffer, 10); break;
} }
} }
if (current == '"') var buffer = _content.Substring(start, _index - start);
return isFloat
? new FloatLiteralToken(CreateSpan(indexStart, lineStart, columnStart), buffer)
: new IntLiteralToken(CreateSpan(indexStart, lineStart, columnStart), buffer, 10);
}
private StringLiteralToken ParseString(int indexStart, int lineStart, int columnStart)
{
Next();
var start = _index;
while (true)
{ {
Next(); if (_index >= _content.Length)
var buffer = string.Empty;
while (true)
{ {
var next = Peek(); throw new CompileException(Diagnostic
if (!next.HasValue) .Error("Unclosed string literal")
{ .At(CreateSpan(_index, _line, _column))
throw new TokenizerException(Diagnostic .Build());
.Error("Unclosed string literal") }
.At(_fileName, _line, _column)
.Build());
}
if (next is '\n') var next = _content[_index];
{
_line += 1;
break;
}
if (next is '"') if (next == '\n')
{ {
Next(); throw new CompileException(Diagnostic
break; .Error("Unclosed string literal (newline found)")
} .At(CreateSpan(_index, _line, _column))
.Build());
}
buffer += next; if (next == '"')
{
var buffer = _content.Substring(start, _index - start);
Next();
return new StringLiteralToken(CreateSpan(indexStart, lineStart, columnStart), buffer);
}
Next();
}
}
private bool TryMatchSymbol(int length, int indexStart, int lineStart, int columnStart, out Token token)
{
token = null!;
if (_index + length > _content.Length)
{
return false;
}
var span = _content.AsSpan(_index, length);
if (span is "true")
{
Next(4);
token = new BoolLiteralToken(CreateSpan(indexStart, lineStart, columnStart), true);
return true;
}
if (span is "false")
{
Next(5);
token = new BoolLiteralToken(CreateSpan(indexStart, lineStart, columnStart), false);
return true;
}
var symbol = length switch
{
8 => span switch
{
"continue" => Symbol.Continue,
_ => Symbol.None
},
6 => span switch
{
"return" => Symbol.Return,
"struct" => Symbol.Struct,
"extern" => Symbol.Extern,
"packed" => Symbol.Packed,
"module" => Symbol.Module,
"export" => Symbol.Export,
_ => Symbol.None
},
5 => span switch
{
"break" => Symbol.Break,
"while" => Symbol.While,
"defer" => Symbol.Defer,
_ => Symbol.None
},
4 => span switch
{
"func" => Symbol.Func,
"else" => Symbol.Else,
"enum" => Symbol.Enum,
_ => Symbol.None
},
3 => span switch
{
"for" => Symbol.For,
"let" => Symbol.Let,
_ => Symbol.None
},
2 => span switch
{
"if" => Symbol.If,
"in" => Symbol.In,
"==" => Symbol.Equal,
"!=" => Symbol.NotEqual,
"<=" => Symbol.LessThanOrEqual,
">=" => Symbol.GreaterThanOrEqual,
"<<" => Symbol.LeftShift,
">>" => Symbol.RightShift,
"&&" => Symbol.And,
"||" => Symbol.Or,
"::" => Symbol.DoubleColon,
_ => Symbol.None
},
1 => span[0] switch
{
':' => Symbol.Colon,
'(' => Symbol.OpenParen,
')' => Symbol.CloseParen,
'{' => Symbol.OpenBrace,
'}' => Symbol.CloseBrace,
'[' => Symbol.OpenBracket,
']' => Symbol.CloseBracket,
',' => Symbol.Comma,
'.' => Symbol.Period,
'=' => Symbol.Assign,
'<' => Symbol.LessThan,
'>' => Symbol.GreaterThan,
'+' => Symbol.Plus,
'-' => Symbol.Minus,
'*' => Symbol.Star,
'/' => Symbol.ForwardSlash,
'!' => Symbol.Bang,
'^' => Symbol.Caret,
'&' => Symbol.Ampersand,
';' => Symbol.Semi,
'%' => Symbol.Percent,
'|' => Symbol.Pipe,
'@' => Symbol.At,
'?' => Symbol.QuestionMark,
'~' => Symbol.Tilde,
_ => Symbol.None
},
_ => Symbol.None
};
if (symbol != Symbol.None)
{
var isAlphaKeyword = char.IsLetter(span[0]);
if (isAlphaKeyword)
{
var nextIdx = _index + length;
if (nextIdx < _content.Length)
{
var nextChar = _content[nextIdx];
if (char.IsLetterOrDigit(nextChar) || nextChar == '_')
{
return false;
}
}
}
Next(length);
token = new SymbolToken(CreateSpan(indexStart, lineStart, columnStart), symbol);
return true;
}
return false;
}
private IdentifierToken ParseIdentifier(int indexStart, int lineStart, int columnStart)
{
var start = _index;
while (_index < _content.Length)
{
var ch = _content[_index];
if (char.IsLetterOrDigit(ch) || ch == '_')
{
Next(); Next();
} }
else
return new StringLiteralToken(CreateSpan(lineStart, columnStart), buffer);
}
foreach (var (pattern, symbol) in OrderedSymbols)
{
for (var i = 0; i < pattern.Length; i++)
{ {
var c = Peek(i); break;
if (!c.HasValue || c.Value != pattern[i]) break;
if (i == pattern.Length - 1)
{
for (var j = 0; j <= i; j++)
{
Next();
}
return new SymbolToken(CreateSpan(lineStart, columnStart), symbol);
}
} }
} }
throw new TokenizerException(Diagnostic.Error($"Unknown token '{current}'").Build()); return new IdentifierToken(CreateSpan(indexStart, lineStart, columnStart), _content.Substring(start, _index - start));
} }
private SourceSpan CreateSpan(int lineStart, int columnStart) private SourceSpan CreateSpan(int indexStart, int lineStart, int columnStart)
{ {
return new SourceSpan(_fileName, new SourceLocation(lineStart, columnStart), new SourceLocation(_line, _column)); return new SourceSpan(_fileName, _content, indexStart, Math.Min(_index, _content.Length), lineStart, columnStart, _line, _column);
} }
private char? Peek(int offset = 0) private void Next(int count = 1)
{ {
if (_index + offset < _content.Length) for (var i = 0; i < count; i++)
{ {
return _content[_index + offset]; if (_index < _content.Length)
{
if (_content[_index] == '\n')
{
_line += 1;
_column = 1;
}
else
{
_column++;
}
}
else
{
_column++;
}
_index++;
} }
return null;
}
private void Next()
{
_index += 1;
_column += 1;
}
}
public class TokenizerException : Exception
{
public Diagnostic Diagnostic { get; }
public TokenizerException(Diagnostic diagnostic) : base(diagnostic.Message)
{
Diagnostic = diagnostic;
} }
} }

View File

@@ -1,10 +1,14 @@
using System.Security.Cryptography; using System.Security.Cryptography;
using System.Text; using System.Text;
namespace NubLang.Ast; namespace NubLang.Types;
public abstract class NubType : IEquatable<NubType> public abstract class NubType : IEquatable<NubType>
{ {
public abstract ulong GetSize();
public abstract ulong GetAlignment();
public abstract bool IsAggregate();
public override bool Equals(object? obj) => obj is NubType other && Equals(other); public override bool Equals(object? obj) => obj is NubType other && Equals(other);
public abstract bool Equals(NubType? other); public abstract bool Equals(NubType? other);
@@ -17,24 +21,36 @@ public abstract class NubType : IEquatable<NubType>
public class NubVoidType : NubType public class NubVoidType : NubType
{ {
public override ulong GetSize() => 8;
public override ulong GetAlignment() => 8;
public override bool IsAggregate() => false;
public override string ToString() => "void"; public override string ToString() => "void";
public override bool Equals(NubType? other) => other is NubVoidType; public override bool Equals(NubType? other) => other is NubVoidType;
public override int GetHashCode() => HashCode.Combine(typeof(NubVoidType)); public override int GetHashCode() => HashCode.Combine(typeof(NubVoidType));
} }
public sealed class NubIntType(bool signed, int width) : NubType public sealed class NubIntType(bool signed, ulong width) : NubType
{ {
public bool Signed { get; } = signed; public bool Signed { get; } = signed;
public int Width { get; } = width; public ulong Width { get; } = width;
public override ulong GetSize() => Width / 8;
public override ulong GetAlignment() => Width / 8;
public override bool IsAggregate() => false;
public override string ToString() => $"{(Signed ? "i" : "u")}{Width}"; public override string ToString() => $"{(Signed ? "i" : "u")}{Width}";
public override bool Equals(NubType? other) => other is NubIntType @int && @int.Width == Width && @int.Signed == Signed; public override bool Equals(NubType? other) => other is NubIntType @int && @int.Width == Width && @int.Signed == Signed;
public override int GetHashCode() => HashCode.Combine(typeof(NubIntType), Signed, Width); public override int GetHashCode() => HashCode.Combine(typeof(NubIntType), Signed, Width);
} }
public sealed class NubFloatType(int width) : NubType public sealed class NubFloatType(ulong width) : NubType
{ {
public int Width { get; } = width; public ulong Width { get; } = width;
public override ulong GetSize() => Width / 8;
public override ulong GetAlignment() => Width / 8;
public override bool IsAggregate() => false;
public override string ToString() => $"f{Width}"; public override string ToString() => $"f{Width}";
public override bool Equals(NubType? other) => other is NubFloatType @float && @float.Width == Width; public override bool Equals(NubType? other) => other is NubFloatType @float && @float.Width == Width;
@@ -43,6 +59,10 @@ public sealed class NubFloatType(int width) : NubType
public class NubBoolType : NubType public class NubBoolType : NubType
{ {
public override ulong GetSize() => 1;
public override ulong GetAlignment() => 1;
public override bool IsAggregate() => false;
public override string ToString() => "bool"; public override string ToString() => "bool";
public override bool Equals(NubType? other) => other is NubBoolType; public override bool Equals(NubType? other) => other is NubBoolType;
public override int GetHashCode() => HashCode.Combine(typeof(NubBoolType)); public override int GetHashCode() => HashCode.Combine(typeof(NubBoolType));
@@ -52,6 +72,10 @@ public sealed class NubPointerType(NubType baseType) : NubType
{ {
public NubType BaseType { get; } = baseType; public NubType BaseType { get; } = baseType;
public override ulong GetSize() => 8;
public override ulong GetAlignment() => 8;
public override bool IsAggregate() => false;
public override string ToString() => "^" + BaseType; public override string ToString() => "^" + BaseType;
public override bool Equals(NubType? other) => other is NubPointerType pointer && BaseType.Equals(pointer.BaseType); public override bool Equals(NubType? other) => other is NubPointerType pointer && BaseType.Equals(pointer.BaseType);
public override int GetHashCode() => HashCode.Combine(typeof(NubPointerType), BaseType); public override int GetHashCode() => HashCode.Combine(typeof(NubPointerType), BaseType);
@@ -62,6 +86,10 @@ public class NubFuncType(List<NubType> parameters, NubType returnType) : NubType
public List<NubType> Parameters { get; } = parameters; public List<NubType> Parameters { get; } = parameters;
public NubType ReturnType { get; } = returnType; public NubType ReturnType { get; } = returnType;
public override ulong GetSize() => 8;
public override ulong GetAlignment() => 8;
public override bool IsAggregate() => false;
public override string ToString() => $"func({string.Join(", ", Parameters)}): {ReturnType}"; public override string ToString() => $"func({string.Join(", ", Parameters)}): {ReturnType}";
public override bool Equals(NubType? other) => other is NubFuncType func && ReturnType.Equals(func.ReturnType) && Parameters.SequenceEqual(func.Parameters); public override bool Equals(NubType? other) => other is NubFuncType func && ReturnType.Equals(func.ReturnType) && Parameters.SequenceEqual(func.Parameters);
@@ -79,12 +107,70 @@ public class NubFuncType(List<NubType> parameters, NubType returnType) : NubType
} }
} }
public class NubStructType(string module, string name, List<NubStructFieldType> fields) : NubType public class NubStructType(string module, string name, bool packed, List<NubStructFieldType> fields) : NubType
{ {
public string Module { get; } = module; public string Module { get; } = module;
public string Name { get; } = name; public string Name { get; } = name;
public bool Packed { get; } = packed;
public List<NubStructFieldType> Fields { get; set; } = fields; public List<NubStructFieldType> Fields { get; set; } = fields;
public int GetFieldIndex(string name)
{
return Fields.FindIndex(x => x.Name == name);
}
public Dictionary<string, ulong> GetFieldOffsets()
{
var offsets = new Dictionary<string, ulong>();
ulong offset = 0;
foreach (var field in Fields)
{
var alignment = Packed ? 1 : field.Type.GetAlignment();
if (!Packed)
{
var padding = (alignment - offset % alignment) % alignment;
offset += padding;
}
offsets[field.Name] = offset;
offset += field.Type.GetSize();
}
return offsets;
}
public override ulong GetSize()
{
var offsets = GetFieldOffsets();
if (Fields.Count == 0)
{
return 0;
}
var lastField = Fields.Last();
var size = offsets[lastField.Name] + lastField.Type.GetSize();
if (!Packed)
{
var structAlignment = GetAlignment();
var padding = (structAlignment - size % structAlignment) % structAlignment;
size += padding;
}
return size;
}
public override ulong GetAlignment()
{
if (Fields.Count == 0)
return 1;
return Packed ? 1 : Fields.Max(f => f.Type.GetAlignment());
}
public override bool IsAggregate() => true;
public override string ToString() => $"{Module}::{Name}"; public override string ToString() => $"{Module}::{Name}";
public override bool Equals(NubType? other) => other is NubStructType structType && Name == structType.Name && Module == structType.Module; public override bool Equals(NubType? other) => other is NubStructType structType && Name == structType.Name && Module == structType.Module;
public override int GetHashCode() => HashCode.Combine(typeof(NubStructType), Module, Name); public override int GetHashCode() => HashCode.Combine(typeof(NubStructType), Module, Name);
@@ -101,15 +187,23 @@ public class NubSliceType(NubType elementType) : NubType
{ {
public NubType ElementType { get; } = elementType; public NubType ElementType { get; } = elementType;
public override ulong GetSize() => 16; // note(nub31): Fat pointer
public override ulong GetAlignment() => 8;
public override bool IsAggregate() => true;
public override string ToString() => "[]" + ElementType; public override string ToString() => "[]" + ElementType;
public override bool Equals(NubType? other) => other is NubSliceType slice && ElementType.Equals(slice.ElementType); public override bool Equals(NubType? other) => other is NubSliceType slice && ElementType.Equals(slice.ElementType);
public override int GetHashCode() => HashCode.Combine(typeof(NubSliceType), ElementType); public override int GetHashCode() => HashCode.Combine(typeof(NubSliceType), ElementType);
} }
public class NubConstArrayType(NubType elementType, long size) : NubType public class NubConstArrayType(NubType elementType, ulong size) : NubType
{ {
public NubType ElementType { get; } = elementType; public NubType ElementType { get; } = elementType;
public long Size { get; } = size; public ulong Size { get; } = size;
public override ulong GetSize() => ElementType.GetSize() * Size;
public override ulong GetAlignment() => ElementType.GetAlignment();
public override bool IsAggregate() => true;
public override string ToString() => $"[{Size}]{ElementType}"; public override string ToString() => $"[{Size}]{ElementType}";
public override bool Equals(NubType? other) => other is NubConstArrayType array && ElementType.Equals(array.ElementType) && Size == array.Size; public override bool Equals(NubType? other) => other is NubConstArrayType array && ElementType.Equals(array.ElementType) && Size == array.Size;
@@ -120,6 +214,10 @@ public class NubArrayType(NubType elementType) : NubType
{ {
public NubType ElementType { get; } = elementType; public NubType ElementType { get; } = elementType;
public override ulong GetSize() => 8;
public override ulong GetAlignment() => 8;
public override bool IsAggregate() => false; // note(nub31): Just a pointer
public override string ToString() => $"[?]{ElementType}"; public override string ToString() => $"[?]{ElementType}";
public override bool Equals(NubType? other) => other is NubArrayType array && ElementType.Equals(array.ElementType); public override bool Equals(NubType? other) => other is NubArrayType array && ElementType.Equals(array.ElementType);
public override int GetHashCode() => HashCode.Combine(typeof(NubArrayType), ElementType); public override int GetHashCode() => HashCode.Combine(typeof(NubArrayType), ElementType);
@@ -127,6 +225,10 @@ public class NubArrayType(NubType elementType) : NubType
public class NubStringType : NubType public class NubStringType : NubType
{ {
public override ulong GetSize() => 16; // note(nub31): Fat pointer
public override ulong GetAlignment() => 8;
public override bool IsAggregate() => true;
public override string ToString() => "string"; public override string ToString() => "string";
public override bool Equals(NubType? other) => other is NubStringType; public override bool Equals(NubType? other) => other is NubStringType;
public override int GetHashCode() => HashCode.Combine(typeof(NubStringType)); public override int GetHashCode() => HashCode.Combine(typeof(NubStringType));

31
compiler/NubLang/Utils.cs Normal file
View File

@@ -0,0 +1,31 @@
namespace NubLang;
public static class Utils
{
public static int LevenshteinDistance(string a, string b)
{
if (a == b) return 0;
if (a.Length == 0) return b.Length;
if (b.Length == 0) return a.Length;
var costs = new int[b.Length + 1];
for (int j = 0; j <= b.Length; j++) costs[j] = j;
for (int i = 1; i <= a.Length; i++)
{
costs[0] = i;
int prevCost = i - 1;
for (int j = 1; j <= b.Length; j++)
{
int currentCost = costs[j];
costs[j] = Math.Min(
Math.Min(costs[j - 1] + 1, costs[j] + 1),
prevCost + (a[i - 1] == b[j - 1] ? 0 : 1)
);
prevCost = currentCost;
}
}
return costs[b.Length];
}
}

View File

@@ -1,4 +1,4 @@
module "main" module main
extern "puts" func puts(text: ^i8) extern "puts" func puts(text: ^i8)

6
examples/playgroud/build.sh Executable file
View File

@@ -0,0 +1,6 @@
#!/bin/bash
set -euo pipefail
nubc main.nub
clang .build/main.ll -o .build/out

View File

@@ -0,0 +1,14 @@
module main
extern "puts" func puts(text: ^i8)
struct Test {
test: ^i8 = "test1"
}
extern "main" func main(argc: i64, argv: [?]^i8)
{
let x = "test"
puts(x)
}

View File

@@ -1,4 +1,4 @@
module "raylib" module raylib
export struct Vector2 export struct Vector2
{ {

View File

@@ -1,11 +1,7 @@
import "raylib" module main
module "main"
extern "main" func main(argc: i64, argv: [?]^i8): i64 extern "main" func main(argc: i64, argv: [?]^i8): i64
{ {
let uwu: []i32 = [1, 2]
raylib::SetConfigFlags(raylib::ConfigFlags.FLAG_VSYNC_HINT | raylib::ConfigFlags.FLAG_WINDOW_RESIZABLE) raylib::SetConfigFlags(raylib::ConfigFlags.FLAG_VSYNC_HINT | raylib::ConfigFlags.FLAG_WINDOW_RESIZABLE)
raylib::InitWindow(1600, 900, "Hi from nub-lang") raylib::InitWindow(1600, 900, "Hi from nub-lang")

1
runtime/.gitignore vendored Normal file
View File

@@ -0,0 +1 @@
.build

6
runtime/build.sh Executable file
View File

@@ -0,0 +1,6 @@
#!/bin/bash
set -euo pipefail
mkdir -p .build
clang -c runtime.c -o .build/runtime.o

49
runtime/ref.c Normal file
View File

@@ -0,0 +1,49 @@
#include "ref.h"
#include <string.h>
#include <stdlib.h>
#include <stdio.h>
void *rc_alloc(size_t size, void (*destructor)(void *self))
{
printf("rc_alloc %zu bytes\n", size);
ref_header *header = malloc(sizeof(ref_header) + size);
memset(header, 0, size);
if (!header)
{
exit(69);
}
header->ref_count = 1;
header->destructor = destructor;
return (void *)(header + 1);
}
void rc_retain(void *obj)
{
if (!obj)
return;
printf("rc_retain\n");
ref_header *header = ((ref_header *)obj) - 1;
header->ref_count++;
}
void rc_release(void *obj)
{
if (!obj)
return;
ref_header *header = ((ref_header *)obj) - 1;
printf("rc_release\n");
if (--header->ref_count == 0)
{
if (header->destructor)
{
header->destructor(obj);
}
free(header);
printf("rc_free\n");
}
}

13
runtime/ref.h Normal file
View File

@@ -0,0 +1,13 @@
#pragma once
#include <stddef.h>
typedef struct
{
int ref_count;
void (*destructor)(void *self);
} ref_header;
void *rc_alloc(size_t size, void (*destructor)(void *self));
void rc_retain(void *obj);
void rc_release(void *obj);

1
runtime/runtime.c Normal file
View File

@@ -0,0 +1 @@
#include "ref.c"

View File

@@ -31,6 +31,12 @@
"configuration": "./language-configuration.json" "configuration": "./language-configuration.json"
} }
], ],
"commands": [
{
"command": "nub.setRootPath",
"title": "Set root path"
}
],
"grammars": [ "grammars": [
{ {
"language": "nub", "language": "nub",

View File

@@ -32,7 +32,19 @@ export async function activate(context: vscode.ExtensionContext) {
} }
); );
vscode.commands.registerCommand('nub.setRootPath', setRootPath);
client.start(); client.start();
const choice = await vscode.window.showInformationMessage(
'Do you want to set the root directory for the project',
'Yes',
'No'
);
if (choice === 'Yes') {
await setRootPath();
}
} }
export function deactivate(): Thenable<void> | undefined { export function deactivate(): Thenable<void> | undefined {
@@ -42,3 +54,25 @@ export function deactivate(): Thenable<void> | undefined {
return client.stop(); return client.stop();
} }
async function setRootPath() {
if (!client) return;
const folder = await vscode.window.showOpenDialog({
canSelectFolders: true,
canSelectFiles: false,
canSelectMany: false,
openLabel: 'Select root location'
});
if (folder && folder.length > 0) {
const newRoot = folder[0].fsPath;
await client.sendRequest('workspace/executeCommand', {
command: 'nub.setRootPath',
arguments: [newRoot]
});
vscode.window.showInformationMessage(`Root path set to: ${newRoot}`);
}
}