Compare commits

...

2 Commits

Author SHA1 Message Date
nub31
828e20ddb6 ... 2025-10-26 14:15:25 +01:00
nub31
396ddf93a2 ... 2025-10-25 18:07:34 +02:00
19 changed files with 952 additions and 599 deletions

View File

@@ -21,7 +21,7 @@ foreach (var file in args)
}
var modules = Module.Collect(syntaxTrees);
var compilationUnits = new List<CompilationUnit>();
var compilationUnits = new List<CompilationUnit?>();
for (var i = 0; i < args.Length; i++)
{
@@ -51,6 +51,11 @@ for (var i = 0; i < args.Length; i++)
var file = args[i];
var compilationUnit = compilationUnits[i];
if (compilationUnit == null)
{
continue;
}
var generator = new Generator(compilationUnit);
var directory = Path.GetDirectoryName(file);
if (!string.IsNullOrWhiteSpace(directory))

View File

@@ -123,21 +123,24 @@ internal class CompletionHandler(WorkspaceManager workspaceManager) : Completion
{
completions.AddRange(_statementSnippets);
foreach (var prototype in compilationUnit.ImportedFunctions)
foreach (var (module, prototypes) in compilationUnit.ImportedFunctions)
{
var parameterStrings = new List<string>();
foreach (var (index, parameter) in prototype.Parameters.Index())
foreach (var prototype in prototypes)
{
parameterStrings.AddRange($"${{{index + 1}:{parameter.Name}}}");
}
var parameterStrings = new List<string>();
foreach (var (index, parameter) in prototype.Parameters.Index())
{
parameterStrings.AddRange($"${{{index + 1}:{parameter.NameToken.Value}}}");
}
completions.Add(new CompletionItem
{
Kind = CompletionItemKind.Function,
Label = $"{prototype.Module}::{prototype.Name}",
InsertTextFormat = InsertTextFormat.Snippet,
InsertText = $"{prototype.Module}::{prototype.Name}({string.Join(", ", parameterStrings)})",
});
completions.Add(new CompletionItem
{
Kind = CompletionItemKind.Function,
Label = $"{module.Value}::{prototype.NameToken.Value}",
InsertTextFormat = InsertTextFormat.Snippet,
InsertText = $"{module.Value}::{prototype.NameToken.Value}({string.Join(", ", parameterStrings)})",
});
}
}
foreach (var parameter in function.Prototype.Parameters)
@@ -145,8 +148,8 @@ internal class CompletionHandler(WorkspaceManager workspaceManager) : Completion
completions.Add(new CompletionItem
{
Kind = CompletionItemKind.Variable,
Label = parameter.Name,
InsertText = parameter.Name,
Label = parameter.NameToken.Value,
InsertText = parameter.NameToken.Value,
});
}
@@ -159,8 +162,8 @@ internal class CompletionHandler(WorkspaceManager workspaceManager) : Completion
completions.Add(new CompletionItem
{
Kind = CompletionItemKind.Variable,
Label = variable.Name,
InsertText = variable.Name,
Label = variable.NameToken.Value,
InsertText = variable.NameToken.Value,
});
}
}

View File

@@ -37,7 +37,7 @@ internal class DefinitionHandler(WorkspaceManager workspaceManager) : Definition
{
var function = compilationUnit.FunctionAtPosition(line, character);
var parameter = function?.Prototype.Parameters.FirstOrDefault(x => x.Name == variableIdentifierNode.Name);
var parameter = function?.Prototype.Parameters.FirstOrDefault(x => x.NameToken.Value == variableIdentifierNode.NameToken.Value);
if (parameter != null)
{
return new LocationOrLocationLinks(parameter.ToLocation());
@@ -46,7 +46,7 @@ internal class DefinitionHandler(WorkspaceManager workspaceManager) : Definition
var variable = function?.Body?
.Descendants()
.OfType<VariableDeclarationNode>()
.FirstOrDefault(x => x.Name == variableIdentifierNode.Name);
.FirstOrDefault(x => x.NameToken.Value == variableIdentifierNode.NameToken.Value);
if (variable != null)
{
@@ -57,7 +57,11 @@ internal class DefinitionHandler(WorkspaceManager workspaceManager) : Definition
}
case FuncIdentifierNode funcIdentifierNode:
{
var prototype = compilationUnit.ImportedFunctions.FirstOrDefault(x => x.Module == funcIdentifierNode.Module && x.Name == funcIdentifierNode.Name);
var prototype = compilationUnit.ImportedFunctions
.Where(x => x.Key.Value == funcIdentifierNode.ModuleToken.Value)
.SelectMany(x => x.Value)
.FirstOrDefault(x => x.NameToken.Value == funcIdentifierNode.NameToken.Value);
if (prototype != null)
{
return new LocationOrLocationLinks(prototype.ToLocation());

View File

@@ -62,10 +62,10 @@ internal class HoverHandler(WorkspaceManager workspaceManager) : HoverHandlerBas
FuncNode funcNode => CreateFuncPrototypeMessage(funcNode.Prototype),
FuncPrototypeNode funcPrototypeNode => CreateFuncPrototypeMessage(funcPrototypeNode),
FuncIdentifierNode funcIdentifierNode => CreateFuncIdentifierMessage(funcIdentifierNode, compilationUnit),
FuncParameterNode funcParameterNode => CreateTypeNameMessage("Function parameter", funcParameterNode.Name, funcParameterNode.Type),
VariableIdentifierNode variableIdentifierNode => CreateTypeNameMessage("Variable", variableIdentifierNode.Name, variableIdentifierNode.Type),
VariableDeclarationNode variableDeclarationNode => CreateTypeNameMessage("Variable declaration", variableDeclarationNode.Name, variableDeclarationNode.Type),
StructFieldAccessNode structFieldAccessNode => CreateTypeNameMessage("Struct field", $"{structFieldAccessNode.Target.Type}.{structFieldAccessNode.Field}", structFieldAccessNode.Type),
FuncParameterNode funcParameterNode => CreateTypeNameMessage("Function parameter", funcParameterNode.NameToken.Value, funcParameterNode.Type),
VariableIdentifierNode variableIdentifierNode => CreateTypeNameMessage("Variable", variableIdentifierNode.NameToken.Value, variableIdentifierNode.Type),
VariableDeclarationNode variableDeclarationNode => CreateTypeNameMessage("Variable declaration", variableDeclarationNode.NameToken.Value, variableDeclarationNode.Type),
StructFieldAccessNode structFieldAccessNode => CreateTypeNameMessage("Struct field", $"{structFieldAccessNode.Target.Type}.{structFieldAccessNode.FieldToken.Value}", structFieldAccessNode.Type),
CStringLiteralNode cStringLiteralNode => CreateLiteralMessage(cStringLiteralNode.Type, '"' + cStringLiteralNode.Value + '"'),
StringLiteralNode stringLiteralNode => CreateLiteralMessage(stringLiteralNode.Type, '"' + stringLiteralNode.Value + '"'),
BoolLiteralNode boolLiteralNode => CreateLiteralMessage(boolLiteralNode.Type, boolLiteralNode.Value.ToString()),
@@ -113,11 +113,15 @@ internal class HoverHandler(WorkspaceManager workspaceManager) : HoverHandlerBas
private static string CreateFuncIdentifierMessage(FuncIdentifierNode funcIdentifierNode, CompilationUnit compilationUnit)
{
var func = compilationUnit.ImportedFunctions.FirstOrDefault(x => x.Module == funcIdentifierNode.Module && x.Name == funcIdentifierNode.Name);
var func = compilationUnit.ImportedFunctions
.Where(x => x.Key.Value == funcIdentifierNode.ModuleToken.Value)
.SelectMany(x => x.Value)
.FirstOrDefault(x => x.NameToken.Value == funcIdentifierNode.NameToken.Value);
if (func == null)
{
return $"""
**Function** `{funcIdentifierNode.Module}::{funcIdentifierNode.Name}`
**Function** `{funcIdentifierNode.ModuleToken.Value}::{funcIdentifierNode.NameToken.Value}`
```nub
// Declaration not found
```
@@ -129,13 +133,13 @@ internal class HoverHandler(WorkspaceManager workspaceManager) : HoverHandlerBas
private static string CreateFuncPrototypeMessage(FuncPrototypeNode funcPrototypeNode)
{
var parameterText = string.Join(", ", funcPrototypeNode.Parameters.Select(x => $"{x.Name}: {x.Type}"));
var externText = funcPrototypeNode.ExternSymbol != null ? $"extern \"{funcPrototypeNode.ExternSymbol}\" " : "";
var parameterText = string.Join(", ", funcPrototypeNode.Parameters.Select(x => $"{x.NameToken.Value}: {x.Type}"));
var externText = funcPrototypeNode.ExternSymbolToken != null ? $"extern \"{funcPrototypeNode.ExternSymbolToken.Value}\" " : "";
return $"""
**Function** `{funcPrototypeNode.Module}::{funcPrototypeNode.Name}`
**Function** `{funcPrototypeNode.NameToken.Value}`
```nub
{externText}func {funcPrototypeNode.Module}::{funcPrototypeNode.Name}({parameterText}): {funcPrototypeNode.ReturnType}
{externText}func {funcPrototypeNode.NameToken.Value}({parameterText}): {funcPrototypeNode.ReturnType}
```
""";
}

View File

@@ -34,7 +34,15 @@ public class WorkspaceManager(DiagnosticsPublisher diagnosticsPublisher)
var typeChecker = new TypeChecker(syntaxTree, modules);
var result = typeChecker.Check();
diagnosticsPublisher.Publish(fsPath, typeChecker.Diagnostics);
_compilationUnits[fsPath] = result;
if (result == null)
{
_compilationUnits.Remove(fsPath);
}
else
{
_compilationUnits[fsPath] = result;
}
}
}
@@ -57,7 +65,15 @@ public class WorkspaceManager(DiagnosticsPublisher diagnosticsPublisher)
var typeChecker = new TypeChecker(syntaxTree, modules);
var result = typeChecker.Check();
diagnosticsPublisher.Publish(fsPath, typeChecker.Diagnostics);
_compilationUnits[fsPath] = result;
if (result == null)
{
_compilationUnits.Remove(fsPath);
}
else
{
_compilationUnits[fsPath] = result;
}
}
public void RemoveFile(DocumentUri path)

View File

@@ -1,15 +1,11 @@
using NubLang.Syntax;
namespace NubLang.Ast;
public sealed class CompilationUnit
public sealed class CompilationUnit(IdentifierToken module, List<FuncNode> functions, Dictionary<IdentifierToken, List<NubStructType>> importedStructTypes, Dictionary<IdentifierToken, List<FuncPrototypeNode>> importedFunctions)
{
public CompilationUnit(List<FuncNode> functions, List<NubStructType> importedStructTypes, List<FuncPrototypeNode> importedFunctions)
{
Functions = functions;
ImportedStructTypes = importedStructTypes;
ImportedFunctions = importedFunctions;
}
public List<FuncNode> Functions { get; }
public List<NubStructType> ImportedStructTypes { get; }
public List<FuncPrototypeNode> ImportedFunctions { get; }
public IdentifierToken Module { get; } = module;
public List<FuncNode> Functions { get; } = functions;
public Dictionary<IdentifierToken, List<NubStructType>> ImportedStructTypes { get; } = importedStructTypes;
public Dictionary<IdentifierToken, List<FuncPrototypeNode>> ImportedFunctions { get; } = importedFunctions;
}

View File

@@ -31,15 +31,14 @@ public abstract class Node(List<Token> tokens)
#region Definitions
public abstract class DefinitionNode(List<Token> tokens, string module, string name) : Node(tokens)
public abstract class DefinitionNode(List<Token> tokens, IdentifierToken nameToken) : Node(tokens)
{
public string Module { get; } = module;
public string Name { get; } = name;
public IdentifierToken NameToken { get; } = nameToken;
}
public class FuncParameterNode(List<Token> tokens, string name, NubType type) : Node(tokens)
public class FuncParameterNode(List<Token> tokens, IdentifierToken nameToken, NubType type) : Node(tokens)
{
public string Name { get; } = name;
public IdentifierToken NameToken { get; } = nameToken;
public NubType Type { get; } = type;
public override IEnumerable<Node> Children()
@@ -48,11 +47,10 @@ public class FuncParameterNode(List<Token> tokens, string name, NubType type) :
}
}
public class FuncPrototypeNode(List<Token> tokens, string module, string name, string? externSymbol, List<FuncParameterNode> parameters, NubType returnType) : Node(tokens)
public class FuncPrototypeNode(List<Token> tokens, IdentifierToken nameToken, StringLiteralToken? externSymbolToken, List<FuncParameterNode> parameters, NubType returnType) : Node(tokens)
{
public string Module { get; } = module;
public string Name { get; } = name;
public string? ExternSymbol { get; } = externSymbol;
public IdentifierToken NameToken { get; } = nameToken;
public StringLiteralToken? ExternSymbolToken { get; } = externSymbolToken;
public List<FuncParameterNode> Parameters { get; } = parameters;
public NubType ReturnType { get; } = returnType;
@@ -62,7 +60,7 @@ public class FuncPrototypeNode(List<Token> tokens, string module, string name, s
}
}
public class FuncNode(List<Token> tokens, FuncPrototypeNode prototype, BlockNode? body) : DefinitionNode(tokens, prototype.Module, prototype.Name)
public class FuncNode(List<Token> tokens, FuncPrototypeNode prototype, BlockNode? body) : DefinitionNode(tokens, prototype.NameToken)
{
public FuncPrototypeNode Prototype { get; } = prototype;
public BlockNode? Body { get; } = body;
@@ -144,9 +142,9 @@ public class IfNode(List<Token> tokens, ExpressionNode condition, BlockNode body
}
}
public class VariableDeclarationNode(List<Token> tokens, string name, ExpressionNode? assignment, NubType type) : StatementNode(tokens)
public class VariableDeclarationNode(List<Token> tokens, IdentifierToken nameToken, ExpressionNode? assignment, NubType type) : StatementNode(tokens)
{
public string Name { get; } = name;
public IdentifierToken NameToken { get; } = nameToken;
public ExpressionNode? Assignment { get; } = assignment;
public NubType Type { get; } = type;
@@ -184,10 +182,10 @@ public class WhileNode(List<Token> tokens, ExpressionNode condition, BlockNode b
}
}
public class ForSliceNode(List<Token> tokens, string elementName, string? indexName, ExpressionNode target, BlockNode body) : StatementNode(tokens)
public class ForSliceNode(List<Token> tokens, IdentifierToken elementNameToken, IdentifierToken? indexNameToken, ExpressionNode target, BlockNode body) : StatementNode(tokens)
{
public string ElementName { get; } = elementName;
public string? IndexName { get; } = indexName;
public IdentifierToken ElementNameToken { get; } = elementNameToken;
public IdentifierToken? IndexNameToken { get; } = indexNameToken;
public ExpressionNode Target { get; } = target;
public BlockNode Body { get; } = body;
@@ -198,10 +196,10 @@ public class ForSliceNode(List<Token> tokens, string elementName, string? indexN
}
}
public class ForConstArrayNode(List<Token> tokens, string elementName, string? indexName, ExpressionNode target, BlockNode body) : StatementNode(tokens)
public class ForConstArrayNode(List<Token> tokens, IdentifierToken elementNameToken, IdentifierToken? indexNameToken, ExpressionNode target, BlockNode body) : StatementNode(tokens)
{
public string ElementName { get; } = elementName;
public string? IndexName { get; } = indexName;
public IdentifierToken ElementNameToken { get; } = elementNameToken;
public IdentifierToken? IndexNameToken { get; } = indexNameToken;
public ExpressionNode Target { get; } = target;
public BlockNode Body { get; } = body;
@@ -385,7 +383,7 @@ public class Float64LiteralNode(List<Token> tokens, double value) : RValueExpres
}
}
public class BoolLiteralNode(List<Token> tokens, NubType type, bool value) : RValueExpressionNode(tokens, type)
public class BoolLiteralNode(List<Token> tokens, bool value) : RValueExpressionNode(tokens, new NubBoolType())
{
public bool Value { get; } = value;
@@ -434,9 +432,9 @@ public class FuncCallNode(List<Token> tokens, NubType type, ExpressionNode expre
}
}
public class VariableIdentifierNode(List<Token> tokens, NubType type, string name) : LValueExpressionNode(tokens, type)
public class VariableIdentifierNode(List<Token> tokens, NubType type, IdentifierToken nameToken) : LValueExpressionNode(tokens, type)
{
public string Name { get; } = name;
public IdentifierToken NameToken { get; } = nameToken;
public override IEnumerable<Node> Children()
{
@@ -444,11 +442,11 @@ public class VariableIdentifierNode(List<Token> tokens, NubType type, string nam
}
}
public class FuncIdentifierNode(List<Token> tokens, NubType type, string module, string name, string? externSymbol) : RValueExpressionNode(tokens, type)
public class FuncIdentifierNode(List<Token> tokens, NubType type, IdentifierToken moduleToken, IdentifierToken nameToken, StringLiteralToken? externSymbolToken) : RValueExpressionNode(tokens, type)
{
public string Module { get; } = module;
public string Name { get; } = name;
public string? ExternSymbol { get; } = externSymbol;
public IdentifierToken ModuleToken { get; } = moduleToken;
public IdentifierToken NameToken { get; } = nameToken;
public StringLiteralToken? ExternSymbolToken { get; } = externSymbolToken;
public override IEnumerable<Node> Children()
{
@@ -522,10 +520,10 @@ public class AddressOfNode(List<Token> tokens, NubType type, LValueExpressionNod
}
}
public class StructFieldAccessNode(List<Token> tokens, NubType type, ExpressionNode target, string field) : LValueExpressionNode(tokens, type)
public class StructFieldAccessNode(List<Token> tokens, NubType type, ExpressionNode target, IdentifierToken fieldToken) : LValueExpressionNode(tokens, type)
{
public ExpressionNode Target { get; } = target;
public string Field { get; } = field;
public IdentifierToken FieldToken { get; } = fieldToken;
public override IEnumerable<Node> Children()
{
@@ -533,9 +531,9 @@ public class StructFieldAccessNode(List<Token> tokens, NubType type, ExpressionN
}
}
public class StructInitializerNode(List<Token> tokens, NubType type, Dictionary<string, ExpressionNode> initializers) : RValueExpressionNode(tokens, type)
public class StructInitializerNode(List<Token> tokens, NubType type, Dictionary<IdentifierToken, ExpressionNode> initializers) : RValueExpressionNode(tokens, type)
{
public Dictionary<string, ExpressionNode> Initializers { get; } = initializers;
public Dictionary<IdentifierToken, ExpressionNode> Initializers { get; } = initializers;
public override IEnumerable<Node> Children()
{
@@ -576,10 +574,10 @@ public class CastNode(List<Token> tokens, NubType type, ExpressionNode value) :
}
}
public class EnumReferenceIntermediateNode(List<Token> tokens, string module, string name) : IntermediateExpression(tokens)
public class EnumReferenceIntermediateNode(List<Token> tokens, IdentifierToken moduleToken, IdentifierToken nameToken) : IntermediateExpression(tokens)
{
public string Module { get; } = module;
public string Name { get; } = name;
public IdentifierToken ModuleToken { get; } = moduleToken;
public IdentifierToken NameToken { get; } = nameToken;
public override IEnumerable<Node> Children()
{

View File

@@ -106,10 +106,10 @@ public class NubSliceType(NubType elementType) : NubType
public override int GetHashCode() => HashCode.Combine(typeof(NubSliceType), ElementType);
}
public class NubConstArrayType(NubType elementType, long size) : NubType
public class NubConstArrayType(NubType elementType, ulong size) : NubType
{
public NubType ElementType { get; } = elementType;
public long Size { get; } = size;
public ulong Size { get; } = size;
public override string ToString() => $"[{Size}]{ElementType}";
public override bool Equals(NubType? other) => other is NubConstArrayType array && ElementType.Equals(array.ElementType) && Size == array.Size;

View File

@@ -7,7 +7,7 @@ namespace NubLang.Ast;
public sealed class TypeChecker
{
private readonly SyntaxTree _syntaxTree;
private readonly Dictionary<string, Module> _importedModules;
private readonly Dictionary<string, Module> _modules;
private readonly Stack<Scope> _scopes = [];
private readonly Dictionary<(string Module, string Name), NubType> _typeCache = new();
@@ -20,22 +20,74 @@ public sealed class TypeChecker
public TypeChecker(SyntaxTree syntaxTree, Dictionary<string, Module> modules)
{
_syntaxTree = syntaxTree;
_importedModules = modules
.Where(x => syntaxTree.Imports.Contains(x.Key) || _syntaxTree.ModuleName == x.Key)
.ToDictionary();
_modules = modules;
}
public CompilationUnit Check()
public CompilationUnit? Check()
{
_scopes.Clear();
_typeCache.Clear();
_resolvingTypes.Clear();
var moduleDeclarations = _syntaxTree.TopLevelSyntaxNodes.OfType<ModuleSyntax>().ToList();
if (moduleDeclarations.Count == 0)
{
Diagnostics.Add(Diagnostic.Error("Missing module declaration").WithHelp("module \"main\"").Build());
return null;
}
if (moduleDeclarations.Count > 1)
{
Diagnostics.Add(Diagnostic.Error("Multiple module declarations").WithHelp("Remove extra module declarations").Build());
}
var moduleName = moduleDeclarations[0].NameToken;
var importDeclarations = _syntaxTree.TopLevelSyntaxNodes.OfType<ImportSyntax>().ToList();
foreach (var importDeclaration in importDeclarations)
{
var name = importDeclaration.NameToken.Value;
var last = importDeclarations.Last(x => x.NameToken.Value == name);
if (importDeclaration != last)
{
Diagnostics.Add(Diagnostic
.Warning($"Module \"{last.NameToken.Value}\" is imported twice")
.WithHelp($"Remove duplicate import \"{last.NameToken.Value}\"")
.At(last)
.Build());
}
var exists = _modules.ContainsKey(name);
if (!exists)
{
var suggestions = _modules.Keys
.Select(m => new { Name = m, Distance = Utils.LevenshteinDistance(name, m) })
.OrderBy(x => x.Distance)
.Take(3)
.Where(x => x.Distance <= 3)
.Select(x => $"\"{x.Name}\"")
.ToArray();
var suggestionText = suggestions.Length != 0
? $"Did you mean {string.Join(", ", suggestions)}?"
: "Check that the module name is correct.";
Diagnostics.Add(Diagnostic
.Error($"Module \"{name}\" does not exist")
.WithHelp(suggestionText)
.At(last)
.Build());
return null;
}
}
var functions = new List<FuncNode>();
using (BeginRootScope(_syntaxTree.ModuleName))
using (BeginRootScope(moduleName))
{
foreach (var funcSyntax in _syntaxTree.Definitions.OfType<FuncSyntax>())
foreach (var funcSyntax in _syntaxTree.TopLevelSyntaxNodes.OfType<FuncSyntax>())
{
try
{
@@ -48,11 +100,14 @@ public sealed class TypeChecker
}
}
var importedStructTypes = new List<NubStructType>();
var importedFunctions = new List<FuncPrototypeNode>();
var importedStructTypes = new Dictionary<IdentifierToken, List<NubStructType>>();
var importedFunctions = new Dictionary<IdentifierToken, List<FuncPrototypeNode>>();
foreach (var (name, module) in _importedModules)
foreach (var (name, module) in GetImportedModules())
{
var moduleStructs = new List<NubStructType>();
var moduleFunctions = new List<FuncPrototypeNode>();
using (BeginRootScope(name))
{
foreach (var structSyntax in module.Structs(true))
@@ -60,10 +115,10 @@ public sealed class TypeChecker
try
{
var fields = structSyntax.Fields
.Select(f => new NubStructFieldType(f.Name, ResolveType(f.Type), f.Value != null))
.Select(f => new NubStructFieldType(f.NameToken.Value, ResolveType(f.Type), f.Value != null))
.ToList();
importedStructTypes.Add(new NubStructType(name, structSyntax.Name, fields));
moduleStructs.Add(new NubStructType(name.Value, structSyntax.NameToken.Value, fields));
}
catch (TypeCheckerException e)
{
@@ -71,21 +126,62 @@ public sealed class TypeChecker
}
}
importedStructTypes[name] = moduleStructs;
foreach (var funcSyntax in module.Functions(true))
{
try
{
importedFunctions.Add(CheckFuncPrototype(funcSyntax.Prototype));
moduleFunctions.Add(CheckFuncPrototype(funcSyntax.Prototype));
}
catch (TypeCheckerException e)
{
Diagnostics.Add(e.Diagnostic);
}
}
importedFunctions[name] = moduleFunctions;
}
}
return new CompilationUnit(functions, importedStructTypes, importedFunctions);
return new CompilationUnit(moduleName, functions, importedStructTypes, importedFunctions);
}
private List<(IdentifierToken Name, Module Module)> GetImportedModules()
{
var currentModule = _syntaxTree.TopLevelSyntaxNodes.OfType<ModuleSyntax>().First().NameToken;
return _syntaxTree.TopLevelSyntaxNodes
.OfType<ImportSyntax>()
.Select(x => (Name: x.NameToken, Module: _modules[x.NameToken.Value]))
.Concat([(Name: currentModule, Module: _modules[currentModule.Value])])
.ToList();
}
private bool IsCurrentModule(IdentifierToken? module)
{
if (module == null)
{
return true;
}
return module.Value == Scope.Module.Value;
}
private Module? GetImportedModule(string module)
{
var currentModule = _syntaxTree.TopLevelSyntaxNodes.OfType<ModuleSyntax>().First().NameToken;
if (module == currentModule.Value)
{
return _modules[currentModule.Value];
}
var import = _syntaxTree.TopLevelSyntaxNodes.OfType<ImportSyntax>().FirstOrDefault(x => x.NameToken.Value == module);
if (import != null)
{
return _modules[import.NameToken.Value];
}
return null;
}
private ScopeDisposer BeginScope()
@@ -94,7 +190,7 @@ public sealed class TypeChecker
return new ScopeDisposer(this);
}
private ScopeDisposer BeginRootScope(string moduleName)
private ScopeDisposer BeginRootScope(IdentifierToken moduleName)
{
_scopes.Push(new Scope(moduleName));
return new ScopeDisposer(this);
@@ -121,7 +217,7 @@ public sealed class TypeChecker
Scope.SetReturnType(prototype.ReturnType);
foreach (var parameter in prototype.Parameters)
{
Scope.DeclareVariable(new Variable(parameter.Name, parameter.Type));
Scope.DeclareVariable(new Variable(parameter.NameToken, parameter.Type));
}
var body = node.Body == null ? null : CheckBlock(node.Body);
@@ -217,14 +313,14 @@ public sealed class TypeChecker
if (type == null)
{
throw new TypeCheckerException(Diagnostic
.Error($"Cannot infer type of variable {statement.Name}")
.Error($"Cannot infer type of variable {statement.NameToken.Value}")
.At(statement)
.Build());
}
Scope.DeclareVariable(new Variable(statement.Name, type));
Scope.DeclareVariable(new Variable(statement.NameToken, type));
return new VariableDeclarationNode(statement.Tokens, statement.Name, assignmentNode, type);
return new VariableDeclarationNode(statement.Tokens, statement.NameToken, assignmentNode, type);
}
private WhileNode CheckWhile(WhileSyntax statement)
@@ -245,28 +341,28 @@ public sealed class TypeChecker
{
using (BeginScope())
{
Scope.DeclareVariable(new Variable(forSyntax.ElementName, sliceType.ElementType));
if (forSyntax.IndexName != null)
Scope.DeclareVariable(new Variable(forSyntax.ElementNameToken, sliceType.ElementType));
if (forSyntax.IndexNameToken != null)
{
Scope.DeclareVariable(new Variable(forSyntax.IndexName, new NubIntType(false, 64)));
Scope.DeclareVariable(new Variable(forSyntax.IndexNameToken, new NubIntType(false, 64)));
}
var body = CheckBlock(forSyntax.Body);
return new ForSliceNode(forSyntax.Tokens, forSyntax.ElementName, forSyntax.IndexName, target, body);
return new ForSliceNode(forSyntax.Tokens, forSyntax.ElementNameToken, forSyntax.IndexNameToken, target, body);
}
}
case NubConstArrayType constArrayType:
{
using (BeginScope())
{
Scope.DeclareVariable(new Variable(forSyntax.ElementName, constArrayType.ElementType));
if (forSyntax.IndexName != null)
Scope.DeclareVariable(new Variable(forSyntax.ElementNameToken, constArrayType.ElementType));
if (forSyntax.IndexNameToken != null)
{
Scope.DeclareVariable(new Variable(forSyntax.IndexName, new NubIntType(false, 64)));
Scope.DeclareVariable(new Variable(forSyntax.IndexNameToken, new NubIntType(false, 64)));
}
var body = CheckBlock(forSyntax.Body);
return new ForConstArrayNode(forSyntax.Tokens, forSyntax.ElementName, forSyntax.IndexName, target, body);
return new ForConstArrayNode(forSyntax.Tokens, forSyntax.ElementNameToken, forSyntax.IndexNameToken, target, body);
}
}
default:
@@ -284,10 +380,10 @@ public sealed class TypeChecker
var parameters = new List<FuncParameterNode>();
foreach (var parameter in statement.Parameters)
{
parameters.Add(new FuncParameterNode(parameter.Tokens, parameter.Name, ResolveType(parameter.Type)));
parameters.Add(new FuncParameterNode(parameter.Tokens, parameter.NameToken, ResolveType(parameter.Type)));
}
return new FuncPrototypeNode(statement.Tokens, Scope.Module, statement.Name, statement.ExternSymbol, parameters, ResolveType(statement.ReturnType));
return new FuncPrototypeNode(statement.Tokens, statement.NameToken, statement.ExternSymbolToken, parameters, ResolveType(statement.ReturnType));
}
private ExpressionNode CheckExpression(ExpressionSyntax node, NubType? expectedType = null)
@@ -480,7 +576,7 @@ public sealed class TypeChecker
{
NubArrayType => new ArrayInitializerNode(expression.Tokens, new NubArrayType(elementType), values),
NubConstArrayType constArrayType => new ConstArrayInitializerNode(expression.Tokens, constArrayType, values),
_ => new ConstArrayInitializerNode(expression.Tokens, new NubConstArrayType(elementType, expression.Values.Count), values)
_ => new ConstArrayInitializerNode(expression.Tokens, new NubConstArrayType(elementType, (ulong)expression.Values.Count), values)
};
}
@@ -747,81 +843,80 @@ public sealed class TypeChecker
return new FuncCallNode(expression.Tokens, funcType.ReturnType, accessor, parameters);
}
private ExpressionNode? CheckIdentifier(ExpressionSyntax expression, string moduleName, string name)
{
if (!_importedModules.TryGetValue(moduleName, out var module))
{
throw new TypeCheckerException(Diagnostic
.Error($"Module {moduleName} not found")
.WithHelp($"import \"{moduleName}\"")
.At(expression)
.Build());
}
var function = module.Functions(IsCurretModule(moduleName)).FirstOrDefault(x => x.Name == name);
if (function != null)
{
using (BeginRootScope(moduleName))
{
var parameters = function.Prototype.Parameters.Select(x => ResolveType(x.Type)).ToList();
var type = new NubFuncType(parameters, ResolveType(function.Prototype.ReturnType));
return new FuncIdentifierNode(expression.Tokens, type, moduleName, name, function.Prototype.ExternSymbol);
}
}
var enumDef = module.Enums(IsCurretModule(moduleName)).FirstOrDefault(x => x.Name == name);
if (enumDef != null)
{
return new EnumReferenceIntermediateNode(expression.Tokens, moduleName, name);
}
return null;
}
private ExpressionNode CheckLocalIdentifier(LocalIdentifierSyntax expression, NubType? _)
{
// note(nub31): Local identifiers can be variables or a symbol in a module
var scopeIdent = Scope.LookupVariable(expression.Name);
var scopeIdent = Scope.LookupVariable(expression.NameToken);
if (scopeIdent != null)
{
return new VariableIdentifierNode(expression.Tokens, scopeIdent.Type, expression.Name);
return new VariableIdentifierNode(expression.Tokens, scopeIdent.Type, expression.NameToken);
}
var ident = CheckIdentifier(expression, Scope.Module, expression.Name);
if (ident == null)
var module = GetImportedModule(Scope.Module.Value)!;
var function = module.Functions(true).FirstOrDefault(x => x.NameToken.Value == expression.NameToken.Value);
if (function != null)
{
throw new TypeCheckerException(Diagnostic
.Error($"There is no identifier named {expression.Name}")
.At(expression)
.Build());
var parameters = function.Prototype.Parameters.Select(x => ResolveType(x.Type)).ToList();
var type = new NubFuncType(parameters, ResolveType(function.Prototype.ReturnType));
return new FuncIdentifierNode(expression.Tokens, type, Scope.Module, expression.NameToken, function.Prototype.ExternSymbolToken);
}
return ident;
var enumDef = module.Enums(true).FirstOrDefault(x => x.NameToken.Value == expression.NameToken.Value);
if (enumDef != null)
{
return new EnumReferenceIntermediateNode(expression.Tokens, Scope.Module, expression.NameToken);
}
throw new TypeCheckerException(Diagnostic
.Error($"There is no identifier named {expression.NameToken.Value}")
.At(expression)
.Build());
}
private ExpressionNode CheckModuleIdentifier(ModuleIdentifierSyntax expression, NubType? _)
{
// note(nub31): Unlike local identifiers, module identifiers does not look for local variables
var ident = CheckIdentifier(expression, expression.Module, expression.Name);
if (ident == null)
var module = GetImportedModule(expression.ModuleToken.Value);
if (module == null)
{
throw new TypeCheckerException(Diagnostic
.Error($"Module {expression.Module} does not export a member named {expression.Name}")
.At(expression)
.Error($"Module {expression.ModuleToken.Value} not found")
.WithHelp($"import \"{expression.ModuleToken.Value}\"")
.At(expression.ModuleToken)
.Build());
}
return ident;
var function = module.Functions(false).FirstOrDefault(x => x.NameToken.Value == expression.NameToken.Value);
if (function != null)
{
using (BeginRootScope(expression.ModuleToken))
{
var parameters = function.Prototype.Parameters.Select(x => ResolveType(x.Type)).ToList();
var type = new NubFuncType(parameters, ResolveType(function.Prototype.ReturnType));
return new FuncIdentifierNode(expression.Tokens, type, expression.ModuleToken, expression.NameToken, function.Prototype.ExternSymbolToken);
}
}
var enumDef = module.Enums(false).FirstOrDefault(x => x.NameToken.Value == expression.NameToken.Value);
if (enumDef != null)
{
return new EnumReferenceIntermediateNode(expression.Tokens, expression.ModuleToken, expression.NameToken);
}
throw new TypeCheckerException(Diagnostic
.Error($"Module {expression.ModuleToken.Value} does not export a member named {expression.NameToken.Value}")
.At(expression)
.Build());
}
private ExpressionNode CheckStringLiteral(StringLiteralSyntax expression, NubType? expectedType)
{
if (expectedType is NubPointerType { BaseType: NubIntType { Signed: true, Width: 8 } })
{
return new CStringLiteralNode(expression.Tokens, expression.Value);
return new CStringLiteralNode(expression.Tokens, expression.Token.Value);
}
return new StringLiteralNode(expression.Tokens, expression.Value);
return new StringLiteralNode(expression.Tokens, expression.Token.Value);
}
private ExpressionNode CheckIntLiteral(IntLiteralSyntax expression, NubType? expectedType)
@@ -830,10 +925,10 @@ public sealed class TypeChecker
{
return intType.Width switch
{
8 => intType.Signed ? new I8LiteralNode(expression.Tokens, Convert.ToSByte(expression.Value, expression.Base)) : new U8LiteralNode(expression.Tokens, Convert.ToByte(expression.Value, expression.Base)),
16 => intType.Signed ? new I16LiteralNode(expression.Tokens, Convert.ToInt16(expression.Value, expression.Base)) : new U16LiteralNode(expression.Tokens, Convert.ToUInt16(expression.Value, expression.Base)),
32 => intType.Signed ? new I32LiteralNode(expression.Tokens, Convert.ToInt32(expression.Value, expression.Base)) : new U32LiteralNode(expression.Tokens, Convert.ToUInt32(expression.Value, expression.Base)),
64 => intType.Signed ? new I64LiteralNode(expression.Tokens, Convert.ToInt64(expression.Value, expression.Base)) : new U64LiteralNode(expression.Tokens, Convert.ToUInt64(expression.Value, expression.Base)),
8 => intType.Signed ? new I8LiteralNode(expression.Tokens, expression.Token.AsI8) : new U8LiteralNode(expression.Tokens, expression.Token.AsU8),
16 => intType.Signed ? new I16LiteralNode(expression.Tokens, expression.Token.AsI16) : new U16LiteralNode(expression.Tokens, expression.Token.AsU16),
32 => intType.Signed ? new I32LiteralNode(expression.Tokens, expression.Token.AsI32) : new U32LiteralNode(expression.Tokens, expression.Token.AsU32),
64 => intType.Signed ? new I64LiteralNode(expression.Tokens, expression.Token.AsI64) : new U64LiteralNode(expression.Tokens, expression.Token.AsU64),
_ => throw new ArgumentOutOfRangeException()
};
}
@@ -842,13 +937,13 @@ public sealed class TypeChecker
{
return floatType.Width switch
{
32 => new Float32LiteralNode(expression.Tokens, Convert.ToSingle(expression.Value)),
64 => new Float64LiteralNode(expression.Tokens, Convert.ToDouble(expression.Value)),
32 => new Float32LiteralNode(expression.Tokens, expression.Token.AsF32),
64 => new Float64LiteralNode(expression.Tokens, expression.Token.AsF64),
_ => throw new ArgumentOutOfRangeException()
};
}
return new I64LiteralNode(expression.Tokens, Convert.ToInt64(expression.Value, expression.Base));
return new I64LiteralNode(expression.Tokens, expression.Token.AsI64);
}
private ExpressionNode CheckFloatLiteral(FloatLiteralSyntax expression, NubType? expectedType)
@@ -857,18 +952,18 @@ public sealed class TypeChecker
{
return floatType.Width switch
{
32 => new Float32LiteralNode(expression.Tokens, Convert.ToSingle(expression.Value)),
64 => new Float64LiteralNode(expression.Tokens, Convert.ToDouble(expression.Value)),
32 => new Float32LiteralNode(expression.Tokens, expression.Token.AsF32),
64 => new Float64LiteralNode(expression.Tokens, expression.Token.AsF64),
_ => throw new ArgumentOutOfRangeException()
};
}
return new Float64LiteralNode(expression.Tokens, Convert.ToDouble(expression.Value));
return new Float64LiteralNode(expression.Tokens, expression.Token.AsF64);
}
private BoolLiteralNode CheckBoolLiteral(BoolLiteralSyntax expression, NubType? _)
{
return new BoolLiteralNode(expression.Tokens, new NubBoolType(), expression.Value);
return new BoolLiteralNode(expression.Tokens, expression.Token.Value);
}
private ExpressionNode CheckMemberAccess(MemberAccessSyntax expression, NubType? _)
@@ -877,15 +972,17 @@ public sealed class TypeChecker
if (target is EnumReferenceIntermediateNode enumReferenceIntermediate)
{
var enumDef = _importedModules[enumReferenceIntermediate.Module]
.Enums(IsCurretModule(enumReferenceIntermediate.Module))
.First(x => x.Name == enumReferenceIntermediate.Name);
var enumDef = GetImportedModules()
.First(x => x.Name.Value == enumReferenceIntermediate.ModuleToken.Value)
.Module
.Enums(IsCurrentModule(enumReferenceIntermediate.ModuleToken))
.First(x => x.NameToken.Value == enumReferenceIntermediate.NameToken.Value);
var field = enumDef.Fields.FirstOrDefault(x => x.Name == expression.Member);
var field = enumDef.Fields.FirstOrDefault(x => x.NameToken.Value == expression.MemberToken.Value);
if (field == null)
{
throw new TypeCheckerException(Diagnostic
.Error($"Enum {Scope.Module}::{enumReferenceIntermediate.Name} does not have a field named {expression.Member}")
.Error($"Enum {Scope.Module.Value}::{enumReferenceIntermediate.NameToken.Value} does not have a field named {expression.MemberToken.Value}")
.At(enumDef)
.Build());
}
@@ -896,36 +993,96 @@ public sealed class TypeChecker
throw new TypeCheckerException(Diagnostic.Error("Enum type must be an int type").At(enumDef.Type).Build());
}
return enumIntType.Width switch
if (enumIntType.Signed)
{
8 => enumIntType.Signed ? new I8LiteralNode(expression.Tokens, (sbyte)field.Value) : new U8LiteralNode(expression.Tokens, (byte)field.Value),
16 => enumIntType.Signed ? new I16LiteralNode(expression.Tokens, (short)field.Value) : new U16LiteralNode(expression.Tokens, (ushort)field.Value),
32 => enumIntType.Signed ? new I32LiteralNode(expression.Tokens, (int)field.Value) : new U32LiteralNode(expression.Tokens, (uint)field.Value),
64 => enumIntType.Signed ? new I64LiteralNode(expression.Tokens, field.Value) : new U64LiteralNode(expression.Tokens, (ulong)field.Value),
_ => throw new ArgumentOutOfRangeException()
};
var fieldValue = CalculateSignedEnumFieldValue(enumDef, field);
return enumIntType.Width switch
{
8 => new I8LiteralNode(expression.Tokens, (sbyte)fieldValue),
16 => new I16LiteralNode(expression.Tokens, (short)fieldValue),
32 => new I32LiteralNode(expression.Tokens, (int)fieldValue),
64 => new I64LiteralNode(expression.Tokens, fieldValue),
_ => throw new ArgumentOutOfRangeException()
};
}
else
{
var fieldValue = CalculateUnsignedEnumFieldValue(enumDef, field);
return enumIntType.Width switch
{
8 => new U8LiteralNode(expression.Tokens, (byte)fieldValue),
16 => new U16LiteralNode(expression.Tokens, (ushort)fieldValue),
32 => new U32LiteralNode(expression.Tokens, (uint)fieldValue),
64 => new U64LiteralNode(expression.Tokens, fieldValue),
_ => throw new ArgumentOutOfRangeException()
};
}
}
if (target.Type is NubStructType structType)
{
var field = structType.Fields.FirstOrDefault(x => x.Name == expression.Member);
var field = structType.Fields.FirstOrDefault(x => x.Name == expression.MemberToken.Value);
if (field == null)
{
throw new TypeCheckerException(Diagnostic
.Error($"Struct {target.Type} does not have a field with the name {expression.Member}")
.Error($"Struct {target.Type} does not have a field with the name {expression.MemberToken.Value}")
.At(expression)
.Build());
}
return new StructFieldAccessNode(expression.Tokens, field.Type, target, expression.Member);
return new StructFieldAccessNode(expression.Tokens, field.Type, target, expression.MemberToken);
}
throw new TypeCheckerException(Diagnostic
.Error($"Cannot access struct member {expression.Member} on type {target.Type}")
.Error($"Cannot access struct member {expression.MemberToken.Value} on type {target.Type}")
.At(expression)
.Build());
}
private static long CalculateSignedEnumFieldValue(EnumSyntax enumDef, EnumFieldSyntax field)
{
long currentValue = 0;
foreach (var f in enumDef.Fields)
{
if (f.ValueToken != null)
{
currentValue = f.ValueToken.AsI64;
}
if (f == field)
{
return currentValue;
}
currentValue++;
}
throw new UnreachableException();
}
private static ulong CalculateUnsignedEnumFieldValue(EnumSyntax enumDef, EnumFieldSyntax field)
{
ulong currentValue = 0;
foreach (var f in enumDef.Fields)
{
if (f.ValueToken != null)
{
currentValue = f.ValueToken.AsU64;
}
if (f == field)
{
return currentValue;
}
currentValue++;
}
throw new UnreachableException();
}
private StructInitializerNode CheckStructInitializer(StructInitializerSyntax expression, NubType? expectedType)
{
NubStructType? structType = null;
@@ -954,11 +1111,11 @@ public sealed class TypeChecker
.Build());
}
var initializers = new Dictionary<string, ExpressionNode>();
var initializers = new Dictionary<IdentifierToken, ExpressionNode>();
foreach (var initializer in expression.Initializers)
{
var typeField = structType.Fields.FirstOrDefault(x => x.Name == initializer.Key);
var typeField = structType.Fields.FirstOrDefault(x => x.Name == initializer.Key.Value);
if (typeField == null)
{
Diagnostics.AddRange(Diagnostic
@@ -973,7 +1130,7 @@ public sealed class TypeChecker
}
var missingFields = structType.Fields
.Where(x => !x.HasDefaultValue && !initializers.ContainsKey(x.Name))
.Where(x => !x.HasDefaultValue && initializers.All(y => y.Key.Value != x.Name))
.Select(x => x.Name)
.ToArray();
@@ -1049,25 +1206,26 @@ public sealed class TypeChecker
private NubType ResolveCustomType(CustomTypeSyntax customType)
{
if (!_importedModules.TryGetValue(customType.Module ?? Scope.Module, out var module))
var module = GetImportedModule(customType.ModuleToken?.Value ?? Scope.Module.Value);
if (module == null)
{
throw new TypeCheckerException(Diagnostic
.Error($"Module {customType.Module ?? Scope.Module} not found")
.WithHelp($"import \"{customType.Module ?? Scope.Module}\"")
.Error($"Module {customType.ModuleToken?.Value ?? Scope.Module.Value} not found")
.WithHelp($"import \"{customType.ModuleToken?.Value ?? Scope.Module.Value}\"")
.At(customType)
.Build());
}
var enumDef = module.Enums(IsCurretModule(customType.Module)).FirstOrDefault(x => x.Name == customType.Name);
var enumDef = module.Enums(IsCurrentModule(customType.ModuleToken)).FirstOrDefault(x => x.NameToken.Value == customType.NameToken.Value);
if (enumDef != null)
{
return enumDef.Type != null ? ResolveType(enumDef.Type) : new NubIntType(false, 64);
}
var structDef = module.Structs(IsCurretModule(customType.Module)).FirstOrDefault(x => x.Name == customType.Name);
var structDef = module.Structs(IsCurrentModule(customType.ModuleToken)).FirstOrDefault(x => x.NameToken.Value == customType.NameToken.Value);
if (structDef != null)
{
var key = (customType.Module ?? Scope.Module, customType.Name);
var key = (customType.ModuleToken?.Value ?? Scope.Module.Value, customType.NameToken.Value);
if (_typeCache.TryGetValue(key, out var cachedType))
{
@@ -1076,18 +1234,18 @@ public sealed class TypeChecker
if (!_resolvingTypes.Add(key))
{
var placeholder = new NubStructType(customType.Module ?? Scope.Module, customType.Name, []);
var placeholder = new NubStructType(customType.ModuleToken?.Value ?? Scope.Module.Value, customType.NameToken.Value, []);
_typeCache[key] = placeholder;
return placeholder;
}
try
{
var result = new NubStructType(customType.Module ?? Scope.Module, structDef.Name, []);
var result = new NubStructType(customType.ModuleToken?.Value ?? Scope.Module.Value, structDef.NameToken.Value, []);
_typeCache[key] = result;
var fields = structDef.Fields
.Select(x => new NubStructFieldType(x.Name, ResolveType(x.Type), x.Value != null))
.Select(x => new NubStructFieldType(x.NameToken.Value, ResolveType(x.Type), x.Value != null))
.ToList();
result.Fields.AddRange(fields);
@@ -1100,29 +1258,19 @@ public sealed class TypeChecker
}
throw new TypeCheckerException(Diagnostic
.Error($"Type {customType.Name} not found in module {customType.Module ?? Scope.Module}")
.Error($"Type {customType.NameToken.Value} not found in module {customType.ModuleToken?.Value ?? Scope.Module.Value}")
.At(customType)
.Build());
}
private bool IsCurretModule(string? module)
{
if (module == null)
{
return true;
}
return module == Scope.Module;
}
}
public record Variable(string Name, NubType Type);
public record Variable(IdentifierToken Name, NubType Type);
public class Scope(string module, Scope? parent = null)
public class Scope(IdentifierToken module, Scope? parent = null)
{
private NubType? _returnType;
private readonly List<Variable> _variables = [];
public string Module { get; } = module;
public IdentifierToken Module { get; } = module;
public void DeclareVariable(Variable variable)
{
@@ -1139,9 +1287,9 @@ public class Scope(string module, Scope? parent = null)
return _returnType ?? parent?.GetReturnType();
}
public Variable? LookupVariable(string name)
public Variable? LookupVariable(IdentifierToken name)
{
var variable = _variables.FirstOrDefault(x => x.Name == name);
var variable = _variables.FirstOrDefault(x => x.Name.Value == name.Value);
if (variable != null)
{
return variable;

View File

@@ -46,33 +46,39 @@ public class Generator
""");
foreach (var structType in _compilationUnit.ImportedStructTypes)
foreach (var (_, structTypes) in _compilationUnit.ImportedStructTypes)
{
_writer.WriteLine(CType.Create(structType));
_writer.WriteLine("{");
using (_writer.Indent())
foreach (var structType in structTypes)
{
foreach (var field in structType.Fields)
_writer.WriteLine(CType.Create(structType));
_writer.WriteLine("{");
using (_writer.Indent())
{
_writer.WriteLine($"{CType.Create(field.Type, field.Name, constArraysAsPointers: false)};");
foreach (var field in structType.Fields)
{
_writer.WriteLine($"{CType.Create(field.Type, field.Name, constArraysAsPointers: false)};");
}
}
}
_writer.WriteLine("};");
_writer.WriteLine();
_writer.WriteLine("};");
_writer.WriteLine();
}
}
// note(nub31): Forward declarations
foreach (var prototype in _compilationUnit.ImportedFunctions)
foreach (var (module, prototypes) in _compilationUnit.ImportedFunctions)
{
EmitLine(prototype.Tokens.FirstOrDefault());
var parameters = prototype.Parameters.Count != 0
? string.Join(", ", prototype.Parameters.Select(x => CType.Create(x.Type, x.Name)))
: "void";
foreach (var prototype in prototypes)
{
EmitLine(prototype.Tokens.FirstOrDefault());
var parameters = prototype.Parameters.Count != 0
? string.Join(", ", prototype.Parameters.Select(x => CType.Create(x.Type, x.NameToken.Value)))
: "void";
var name = FuncName(prototype.Module, prototype.Name, prototype.ExternSymbol);
_writer.WriteLine($"{CType.Create(prototype.ReturnType, name)}({parameters});");
_writer.WriteLine();
var name = FuncName(module.Value, prototype.NameToken.Value, prototype.ExternSymbolToken?.Value);
_writer.WriteLine($"{CType.Create(prototype.ReturnType, name)}({parameters});");
_writer.WriteLine();
}
}
// note(nub31): Normal functions
@@ -82,10 +88,10 @@ public class Generator
EmitLine(funcNode.Tokens.FirstOrDefault());
var parameters = funcNode.Prototype.Parameters.Count != 0
? string.Join(", ", funcNode.Prototype.Parameters.Select(x => CType.Create(x.Type, x.Name)))
? string.Join(", ", funcNode.Prototype.Parameters.Select(x => CType.Create(x.Type, x.NameToken.Value)))
: "void";
var name = FuncName(funcNode.Module, funcNode.Name, funcNode.Prototype.ExternSymbol);
var name = FuncName(_compilationUnit.Module.Value, funcNode.NameToken.Value, funcNode.Prototype.ExternSymbolToken?.Value);
_writer.WriteLine($"{CType.Create(funcNode.Prototype.ReturnType, name)}({parameters})");
_writer.WriteLine("{");
using (_writer.Indent())
@@ -188,13 +194,13 @@ public class Generator
{
var targetType = (NubSliceType)forSliceNode.Target.Type;
var target = EmitExpression(forSliceNode.Target);
var indexName = forSliceNode.IndexName ?? NewTmp();
var indexName = forSliceNode.IndexNameToken?.Value ?? NewTmp();
_writer.WriteLine($"for (unsigned long long {indexName} = 0; {indexName} < {target}.length; ++{indexName})");
_writer.WriteLine("{");
using (_writer.Indent())
{
_writer.WriteLine($"{CType.Create(targetType.ElementType, forSliceNode.ElementName)} = (({CType.Create(targetType.ElementType)}*){target}.data)[{indexName}];");
_writer.WriteLine($"{CType.Create(targetType.ElementType, forSliceNode.ElementNameToken.Value)} = (({CType.Create(targetType.ElementType)}*){target}.data)[{indexName}];");
EmitBlock(forSliceNode.Body);
}
@@ -205,13 +211,13 @@ public class Generator
{
var targetType = (NubConstArrayType)forConstArrayNode.Target.Type;
var target = EmitExpression(forConstArrayNode.Target);
var indexName = forConstArrayNode.IndexName ?? NewTmp();
var indexName = forConstArrayNode.IndexNameToken?.Value ?? NewTmp();
_writer.WriteLine($"for (unsigned long long {indexName} = 0; {indexName} < {targetType.Size}; ++{indexName})");
_writer.WriteLine("{");
using (_writer.Indent())
{
_writer.WriteLine($"{CType.Create(targetType.ElementType, forConstArrayNode.ElementName)} = {target}[{indexName}];");
_writer.WriteLine($"{CType.Create(targetType.ElementType, forConstArrayNode.ElementNameToken.Value)} = {target}[{indexName}];");
EmitBlock(forConstArrayNode.Body);
}
@@ -295,11 +301,11 @@ public class Generator
if (variableDeclarationNode.Assignment != null)
{
var value = EmitExpression(variableDeclarationNode.Assignment);
_writer.WriteLine($"{CType.Create(variableDeclarationNode.Type, variableDeclarationNode.Name)} = {value};");
_writer.WriteLine($"{CType.Create(variableDeclarationNode.Type, variableDeclarationNode.NameToken.Value)} = {value};");
}
else
{
_writer.WriteLine($"{CType.Create(variableDeclarationNode.Type, variableDeclarationNode.Name)};");
_writer.WriteLine($"{CType.Create(variableDeclarationNode.Type, variableDeclarationNode.NameToken.Value)};");
}
}
@@ -337,7 +343,7 @@ public class Generator
Float64LiteralNode float64LiteralNode => EmitFloat64Literal(float64LiteralNode),
CastNode castNode => EmitCast(castNode),
FuncCallNode funcCallNode => EmitFuncCall(funcCallNode),
FuncIdentifierNode funcIdentifierNode => FuncName(funcIdentifierNode.Module, funcIdentifierNode.Name, funcIdentifierNode.ExternSymbol),
FuncIdentifierNode funcIdentifierNode => FuncName(funcIdentifierNode.ModuleToken.Value, funcIdentifierNode.NameToken.Value, funcIdentifierNode.ExternSymbolToken?.Value),
AddressOfNode addressOfNode => EmitAddressOf(addressOfNode),
SizeNode sizeBuiltinNode => $"sizeof({CType.Create(sizeBuiltinNode.TargetType)})",
SliceIndexAccessNode sliceIndexAccessNode => EmitSliceArrayIndexAccess(sliceIndexAccessNode),
@@ -353,7 +359,7 @@ public class Generator
U32LiteralNode u32LiteralNode => EmitU32Literal(u32LiteralNode),
U64LiteralNode u64LiteralNode => EmitU64Literal(u64LiteralNode),
UnaryExpressionNode unaryExpressionNode => EmitUnaryExpression(unaryExpressionNode),
VariableIdentifierNode variableIdentifierNode => variableIdentifierNode.Name,
VariableIdentifierNode variableIdentifierNode => variableIdentifierNode.NameToken.Value,
_ => throw new ArgumentOutOfRangeException(nameof(expressionNode))
};
@@ -501,7 +507,7 @@ public class Generator
private string EmitStructFieldAccess(StructFieldAccessNode structFieldAccessNode)
{
var structExpr = EmitExpression(structFieldAccessNode.Target);
return $"{structExpr}.{structFieldAccessNode.Field}";
return $"{structExpr}.{structFieldAccessNode.FieldToken.Value}";
}
private string EmitStructInitializer(StructInitializerNode structInitializerNode)
@@ -510,7 +516,7 @@ public class Generator
foreach (var initializer in structInitializerNode.Initializers)
{
var value = EmitExpression(initializer.Value);
initValues.Add($".{initializer.Key} = {value}");
initValues.Add($".{initializer.Key.Value} = {value}");
}
var initString = initValues.Count == 0

View File

@@ -7,19 +7,23 @@ public sealed class Module
var modules = new Dictionary<string, Module>();
foreach (var syntaxTree in syntaxTrees)
{
if (!modules.TryGetValue(syntaxTree.ModuleName, out var module))
var moduleDeclaration = syntaxTree.TopLevelSyntaxNodes.OfType<ModuleSyntax>().FirstOrDefault();
if (moduleDeclaration != null)
{
module = new Module();
modules.Add(syntaxTree.ModuleName, module);
}
if (!modules.TryGetValue(moduleDeclaration.NameToken.Value, out var module))
{
module = new Module();
modules.Add(moduleDeclaration.NameToken.Value, module);
}
module._definitions.AddRange(syntaxTree.Definitions);
module._definitions.AddRange(syntaxTree.TopLevelSyntaxNodes);
}
}
return modules;
}
private readonly List<DefinitionSyntax> _definitions = [];
private readonly List<TopLevelSyntaxNode> _definitions = [];
public List<StructSyntax> Structs(bool includePrivate)
{

View File

@@ -19,9 +19,7 @@ public sealed class Parser
_tokens = tokens;
_tokenIndex = 0;
string? moduleName = null;
var imports = new List<string>();
var definitions = new List<DefinitionSyntax>();
var topLevelSyntaxNodes = new List<TopLevelSyntaxNode>();
while (HasToken)
{
@@ -29,53 +27,21 @@ public sealed class Parser
{
var startIndex = _tokenIndex;
if (TryExpectSymbol(Symbol.Import))
{
var name = ExpectStringLiteral();
if (imports.Contains(name.Value))
{
Diagnostics.Add(Diagnostic
.Warning($"Module {name.Value} is imported twice")
.At(name)
.WithHelp($"Remove duplicate import \"{name.Value}\"")
.Build());
}
else
{
imports.Add(name.Value);
}
continue;
}
if (TryExpectSymbol(Symbol.Module))
{
if (moduleName != null)
{
throw new ParseException(Diagnostic
.Error("Module is declared more than once")
.At(CurrentToken)
.WithHelp("Remove duplicate module declaration")
.Build());
}
moduleName = ExpectStringLiteral().Value;
continue;
}
var exported = TryExpectSymbol(Symbol.Export);
if (TryExpectSymbol(Symbol.Extern))
{
var externSymbol = ExpectStringLiteral();
ExpectSymbol(Symbol.Func);
definitions.Add(ParseFunc(startIndex, exported, externSymbol.Value));
topLevelSyntaxNodes.Add(ParseFunc(startIndex, exported, externSymbol));
continue;
}
var keyword = ExpectSymbol();
DefinitionSyntax definition = keyword.Symbol switch
TopLevelSyntaxNode definition = keyword.Symbol switch
{
Symbol.Module => ParseModule(startIndex),
Symbol.Import => ParseImport(startIndex),
Symbol.Func => ParseFunc(startIndex, exported, null),
Symbol.Struct => ParseStruct(startIndex, exported),
Symbol.Enum => ParseEnum(startIndex, exported),
@@ -86,7 +52,7 @@ public sealed class Parser
.Build())
};
definitions.Add(definition);
topLevelSyntaxNodes.Add(definition);
}
catch (ParseException e)
{
@@ -103,7 +69,19 @@ public sealed class Parser
}
}
return new SyntaxTree(definitions, moduleName ?? "default", imports);
return new SyntaxTree(topLevelSyntaxNodes);
}
private ImportSyntax ParseImport(int startIndex)
{
var name = ExpectIdentifier();
return new ImportSyntax(GetTokens(startIndex), name);
}
private ModuleSyntax ParseModule(int startIndex)
{
var name = ExpectIdentifier();
return new ModuleSyntax(GetTokens(startIndex), name);
}
private FuncParameterSyntax ParseFuncParameter()
@@ -113,10 +91,10 @@ public sealed class Parser
ExpectSymbol(Symbol.Colon);
var type = ParseType();
return new FuncParameterSyntax(GetTokens(startIndex), name.Value, type);
return new FuncParameterSyntax(GetTokens(startIndex), name, type);
}
private FuncSyntax ParseFunc(int startIndex, bool exported, string? externSymbol)
private FuncSyntax ParseFunc(int startIndex, bool exported, StringLiteralToken? externSymbol)
{
var name = ExpectIdentifier();
List<FuncParameterSyntax> parameters = [];
@@ -136,7 +114,7 @@ public sealed class Parser
var returnType = TryExpectSymbol(Symbol.Colon) ? ParseType() : new VoidTypeSyntax([]);
var prototype = new FuncPrototypeSyntax(GetTokens(startIndex), name.Value, exported, externSymbol, parameters, returnType);
var prototype = new FuncPrototypeSyntax(GetTokens(startIndex), name, exported, externSymbol, parameters, returnType);
BlockSyntax? body = null;
var bodyStartIndex = _tokenIndex;
@@ -160,7 +138,7 @@ public sealed class Parser
{
var memberStartIndex = _tokenIndex;
var fieldName = ExpectIdentifier().Value;
var fieldName = ExpectIdentifier();
ExpectSymbol(Symbol.Colon);
var fieldType = ParseType();
@@ -174,7 +152,7 @@ public sealed class Parser
fields.Add(new StructFieldSyntax(GetTokens(memberStartIndex), fieldName, fieldType, fieldValue));
}
return new StructSyntax(GetTokens(startIndex), name.Value, exported, fields);
return new StructSyntax(GetTokens(startIndex), name, exported, fields);
}
private EnumSyntax ParseEnum(int startIndex, bool exported)
@@ -192,13 +170,11 @@ public sealed class Parser
ExpectSymbol(Symbol.OpenBrace);
long value = -1;
while (!TryExpectSymbol(Symbol.CloseBrace))
{
var memberStartIndex = _tokenIndex;
var fieldName = ExpectIdentifier().Value;
long fieldValue;
var fieldName = ExpectIdentifier();
IntLiteralToken? value = null;
if (TryExpectSymbol(Symbol.Assign))
{
@@ -210,19 +186,13 @@ public sealed class Parser
.Build());
}
fieldValue = Convert.ToInt64(intLiteralToken.Value, intLiteralToken.Base);
value = fieldValue;
}
else
{
fieldValue = value + 1;
value = fieldValue;
value = intLiteralToken;
}
fields.Add(new EnumFieldSyntax(GetTokens(memberStartIndex), fieldName, fieldValue));
fields.Add(new EnumFieldSyntax(GetTokens(memberStartIndex), fieldName, value));
}
return new EnumSyntax(GetTokens(startIndex), name.Value, exported, type, fields);
return new EnumSyntax(GetTokens(startIndex), name, exported, type, fields);
}
private StatementSyntax ParseStatement()
@@ -267,7 +237,7 @@ public sealed class Parser
private VariableDeclarationSyntax ParseVariableDeclaration(int startIndex)
{
var name = ExpectIdentifier().Value;
var name = ExpectIdentifier();
TypeSyntax? explicitType = null;
if (TryExpectSymbol(Symbol.Colon))
@@ -334,12 +304,12 @@ public sealed class Parser
private ForSyntax ParseFor(int startIndex)
{
var itemName = ExpectIdentifier().Value;
string? indexName = null;
var itemName = ExpectIdentifier();
IdentifierToken? indexName = null;
if (TryExpectSymbol(Symbol.Comma))
{
indexName = ExpectIdentifier().Value;
indexName = ExpectIdentifier();
}
ExpectSymbol(Symbol.In);
@@ -467,10 +437,10 @@ public sealed class Parser
var token = ExpectToken();
var expr = token switch
{
BoolLiteralToken boolLiteral => new BoolLiteralSyntax(GetTokens(startIndex), boolLiteral.Value),
StringLiteralToken stringLiteral => new StringLiteralSyntax(GetTokens(startIndex), stringLiteral.Value),
FloatLiteralToken floatLiteral => new FloatLiteralSyntax(GetTokens(startIndex), floatLiteral.Value),
IntLiteralToken intLiteral => new IntLiteralSyntax(GetTokens(startIndex), intLiteral.Value, intLiteral.Base),
BoolLiteralToken boolLiteral => new BoolLiteralSyntax(GetTokens(startIndex), boolLiteral),
StringLiteralToken stringLiteral => new StringLiteralSyntax(GetTokens(startIndex), stringLiteral),
FloatLiteralToken floatLiteral => new FloatLiteralSyntax(GetTokens(startIndex), floatLiteral),
IntLiteralToken intLiteral => new IntLiteralSyntax(GetTokens(startIndex), intLiteral),
IdentifierToken identifier => ParseIdentifier(startIndex, identifier),
SymbolToken symbolToken => symbolToken.Symbol switch
{
@@ -528,10 +498,10 @@ public sealed class Parser
if (TryExpectSymbol(Symbol.DoubleColon))
{
var name = ExpectIdentifier();
return new ModuleIdentifierSyntax(GetTokens(startIndex), identifier.Value, name.Value);
return new ModuleIdentifierSyntax(GetTokens(startIndex), identifier, name);
}
return new LocalIdentifierSyntax(GetTokens(startIndex), identifier.Value);
return new LocalIdentifierSyntax(GetTokens(startIndex), identifier);
}
private ExpressionSyntax ParseParenthesizedExpression()
@@ -560,7 +530,7 @@ public sealed class Parser
if (TryExpectSymbol(Symbol.Period))
{
var member = ExpectIdentifier().Value;
var member = ExpectIdentifier();
expr = new MemberAccessSyntax(GetTokens(startIndex), expr, member);
continue;
}
@@ -627,12 +597,12 @@ public sealed class Parser
return new StructInitializerSyntax(GetTokens(startIndex), type, initializers);
}
private Dictionary<string, ExpressionSyntax> ParseStructInitializerBody()
private Dictionary<IdentifierToken, ExpressionSyntax> ParseStructInitializerBody()
{
Dictionary<string, ExpressionSyntax> initializers = [];
Dictionary<IdentifierToken, ExpressionSyntax> initializers = [];
while (!TryExpectSymbol(Symbol.CloseBrace))
{
var name = ExpectIdentifier().Value;
var name = ExpectIdentifier();
ExpectSymbol(Symbol.Assign);
var value = ParseExpression();
initializers.Add(name, value);
@@ -732,16 +702,16 @@ public sealed class Parser
return new BoolTypeSyntax(GetTokens(startIndex));
default:
{
string? module = null;
IdentifierToken? module = null;
if (TryExpectSymbol(Symbol.DoubleColon))
{
var customTypeName = ExpectIdentifier();
module = name.Value;
module = name;
name = customTypeName;
}
return new CustomTypeSyntax(GetTokens(startIndex), module, name.Value);
return new CustomTypeSyntax(GetTokens(startIndex), module, name);
}
}
}
@@ -780,7 +750,7 @@ public sealed class Parser
{
ExpectSymbol(Symbol.CloseBracket);
var baseType = ParseType();
return new ConstArrayTypeSyntax(GetTokens(startIndex), baseType, Convert.ToInt64(intLiteral.Value, intLiteral.Base));
return new ConstArrayTypeSyntax(GetTokens(startIndex), baseType, intLiteral.AsU64);
}
else if (TryExpectSymbol(Symbol.QuestionMark))
{
@@ -938,7 +908,7 @@ public sealed class Parser
}
}
public record SyntaxTree(List<DefinitionSyntax> Definitions, string ModuleName, List<string> Imports);
public record SyntaxTree(List<TopLevelSyntaxNode> TopLevelSyntaxNodes);
public class ParseException : Exception
{

View File

@@ -4,21 +4,27 @@ public abstract record SyntaxNode(List<Token> Tokens);
#region Definitions
public abstract record DefinitionSyntax(List<Token> Tokens, string Name, bool Exported) : SyntaxNode(Tokens);
public record TopLevelSyntaxNode(List<Token> Tokens) : SyntaxNode(Tokens);
public record FuncParameterSyntax(List<Token> Tokens, string Name, TypeSyntax Type) : SyntaxNode(Tokens);
public record ModuleSyntax(List<Token> Tokens, IdentifierToken NameToken) : TopLevelSyntaxNode(Tokens);
public record FuncPrototypeSyntax(List<Token> Tokens, string Name, bool Exported, string? ExternSymbol, List<FuncParameterSyntax> Parameters, TypeSyntax ReturnType) : SyntaxNode(Tokens);
public record ImportSyntax(List<Token> Tokens, IdentifierToken NameToken) : TopLevelSyntaxNode(Tokens);
public record FuncSyntax(List<Token> Tokens, FuncPrototypeSyntax Prototype, BlockSyntax? Body) : DefinitionSyntax(Tokens, Prototype.Name, Prototype.Exported);
public abstract record DefinitionSyntax(List<Token> Tokens, IdentifierToken NameToken, bool Exported) : TopLevelSyntaxNode(Tokens);
public record StructFieldSyntax(List<Token> Tokens, string Name, TypeSyntax Type, ExpressionSyntax? Value) : SyntaxNode(Tokens);
public record FuncParameterSyntax(List<Token> Tokens, IdentifierToken NameToken, TypeSyntax Type) : SyntaxNode(Tokens);
public record StructSyntax(List<Token> Tokens, string Name, bool Exported, List<StructFieldSyntax> Fields) : DefinitionSyntax(Tokens, Name, Exported);
public record FuncPrototypeSyntax(List<Token> Tokens, IdentifierToken NameToken, bool Exported, StringLiteralToken? ExternSymbolToken, List<FuncParameterSyntax> Parameters, TypeSyntax ReturnType) : SyntaxNode(Tokens);
public record EnumFieldSyntax(List<Token> Tokens, string Name, long Value) : SyntaxNode(Tokens);
public record FuncSyntax(List<Token> Tokens, FuncPrototypeSyntax Prototype, BlockSyntax? Body) : DefinitionSyntax(Tokens, Prototype.NameToken, Prototype.Exported);
public record EnumSyntax(List<Token> Tokens, string Name, bool Exported, TypeSyntax? Type, List<EnumFieldSyntax> Fields) : DefinitionSyntax(Tokens, Name, Exported);
public record StructFieldSyntax(List<Token> Tokens, IdentifierToken NameToken, TypeSyntax Type, ExpressionSyntax? Value) : SyntaxNode(Tokens);
public record StructSyntax(List<Token> Tokens, IdentifierToken NameToken, bool Exported, List<StructFieldSyntax> Fields) : DefinitionSyntax(Tokens, NameToken, Exported);
public record EnumFieldSyntax(List<Token> Tokens, IdentifierToken NameToken, IntLiteralToken? ValueToken) : SyntaxNode(Tokens);
public record EnumSyntax(List<Token> Tokens, IdentifierToken NameToken, bool Exported, TypeSyntax? Type, List<EnumFieldSyntax> Fields) : DefinitionSyntax(Tokens, NameToken, Exported);
public enum UnaryOperatorSyntax
{
@@ -64,7 +70,7 @@ public record AssignmentSyntax(List<Token> Tokens, ExpressionSyntax Target, Expr
public record IfSyntax(List<Token> Tokens, ExpressionSyntax Condition, BlockSyntax Body, Variant<IfSyntax, BlockSyntax>? Else) : StatementSyntax(Tokens);
public record VariableDeclarationSyntax(List<Token> Tokens, string Name, TypeSyntax? ExplicitType, ExpressionSyntax? Assignment) : StatementSyntax(Tokens);
public record VariableDeclarationSyntax(List<Token> Tokens, IdentifierToken NameToken, TypeSyntax? ExplicitType, ExpressionSyntax? Assignment) : StatementSyntax(Tokens);
public record ContinueSyntax(List<Token> Tokens) : StatementSyntax(Tokens);
@@ -74,7 +80,7 @@ public record DeferSyntax(List<Token> Tokens, StatementSyntax Statement) : State
public record WhileSyntax(List<Token> Tokens, ExpressionSyntax Condition, BlockSyntax Body) : StatementSyntax(Tokens);
public record ForSyntax(List<Token> Tokens, string ElementName, string? IndexName, ExpressionSyntax Target, BlockSyntax Body) : StatementSyntax(Tokens);
public record ForSyntax(List<Token> Tokens, IdentifierToken ElementNameToken, IdentifierToken? IndexNameToken, ExpressionSyntax Target, BlockSyntax Body) : StatementSyntax(Tokens);
#endregion
@@ -88,9 +94,9 @@ public record UnaryExpressionSyntax(List<Token> Tokens, UnaryOperatorSyntax Oper
public record FuncCallSyntax(List<Token> Tokens, ExpressionSyntax Expression, List<ExpressionSyntax> Parameters) : ExpressionSyntax(Tokens);
public record LocalIdentifierSyntax(List<Token> Tokens, string Name) : ExpressionSyntax(Tokens);
public record LocalIdentifierSyntax(List<Token> Tokens, IdentifierToken NameToken) : ExpressionSyntax(Tokens);
public record ModuleIdentifierSyntax(List<Token> Tokens, string Module, string Name) : ExpressionSyntax(Tokens);
public record ModuleIdentifierSyntax(List<Token> Tokens, IdentifierToken ModuleToken, IdentifierToken NameToken) : ExpressionSyntax(Tokens);
public record ArrayInitializerSyntax(List<Token> Tokens, List<ExpressionSyntax> Values) : ExpressionSyntax(Tokens);
@@ -98,17 +104,17 @@ public record ArrayIndexAccessSyntax(List<Token> Tokens, ExpressionSyntax Target
public record AddressOfSyntax(List<Token> Tokens, ExpressionSyntax Target) : ExpressionSyntax(Tokens);
public record IntLiteralSyntax(List<Token> Tokens, string Value, int Base) : ExpressionSyntax(Tokens);
public record IntLiteralSyntax(List<Token> Tokens, IntLiteralToken Token) : ExpressionSyntax(Tokens);
public record StringLiteralSyntax(List<Token> Tokens, string Value) : ExpressionSyntax(Tokens);
public record StringLiteralSyntax(List<Token> Tokens, StringLiteralToken Token) : ExpressionSyntax(Tokens);
public record BoolLiteralSyntax(List<Token> Tokens, bool Value) : ExpressionSyntax(Tokens);
public record BoolLiteralSyntax(List<Token> Tokens, BoolLiteralToken Token) : ExpressionSyntax(Tokens);
public record FloatLiteralSyntax(List<Token> Tokens, string Value) : ExpressionSyntax(Tokens);
public record FloatLiteralSyntax(List<Token> Tokens, FloatLiteralToken Token) : ExpressionSyntax(Tokens);
public record MemberAccessSyntax(List<Token> Tokens, ExpressionSyntax Target, string Member) : ExpressionSyntax(Tokens);
public record MemberAccessSyntax(List<Token> Tokens, ExpressionSyntax Target, IdentifierToken MemberToken) : ExpressionSyntax(Tokens);
public record StructInitializerSyntax(List<Token> Tokens, TypeSyntax? StructType, Dictionary<string, ExpressionSyntax> Initializers) : ExpressionSyntax(Tokens);
public record StructInitializerSyntax(List<Token> Tokens, TypeSyntax? StructType, Dictionary<IdentifierToken, ExpressionSyntax> Initializers) : ExpressionSyntax(Tokens);
public record DereferenceSyntax(List<Token> Tokens, ExpressionSyntax Target) : ExpressionSyntax(Tokens);
@@ -140,8 +146,8 @@ public record SliceTypeSyntax(List<Token> Tokens, TypeSyntax BaseType) : TypeSyn
public record ArrayTypeSyntax(List<Token> Tokens, TypeSyntax BaseType) : TypeSyntax(Tokens);
public record ConstArrayTypeSyntax(List<Token> Tokens, TypeSyntax BaseType, long Size) : TypeSyntax(Tokens);
public record ConstArrayTypeSyntax(List<Token> Tokens, TypeSyntax BaseType, ulong Size) : TypeSyntax(Tokens);
public record CustomTypeSyntax(List<Token> Tokens, string? Module, string Name) : TypeSyntax(Tokens);
public record CustomTypeSyntax(List<Token> Tokens, IdentifierToken? ModuleToken, IdentifierToken NameToken) : TypeSyntax(Tokens);
#endregion

View File

@@ -2,8 +2,68 @@
namespace NubLang.Syntax;
public abstract record Token(SourceSpan Span);
public record IdentifierToken(SourceSpan Span, string Value) : Token(Span)
{
public override string ToString()
{
return Value;
}
}
public record IntLiteralToken(SourceSpan Span, string Value, int Base) : Token(Span)
{
public ulong AsU64 => Convert.ToUInt64(Value, Base);
public long AsI64 => Convert.ToInt64(Value, Base);
public uint AsU32 => Convert.ToUInt32(Value, Base);
public int AsI32 => Convert.ToInt32(Value, Base);
public ushort AsU16 => Convert.ToUInt16(Value, Base);
public short AsI16 => Convert.ToInt16(Value, Base);
public byte AsU8 => Convert.ToByte(Value, Base);
public sbyte AsI8 => Convert.ToSByte(Value, Base);
public float AsF32 => Convert.ToSingle(AsI32);
public double AsF64 => Convert.ToDouble(AsI64);
public override string ToString()
{
return Value;
}
}
public record StringLiteralToken(SourceSpan Span, string Value) : Token(Span)
{
public override string ToString()
{
return $"\"{Value}\"";
}
}
public record BoolLiteralToken(SourceSpan Span, bool Value) : Token(Span)
{
public override string ToString()
{
return Value ? "true" : "false";
}
}
public record FloatLiteralToken(SourceSpan Span, string Value) : Token(Span)
{
public float AsF32 => Convert.ToSingle(Value);
public double AsF64 => Convert.ToDouble(Value);
public override string ToString()
{
return Value;
}
}
public enum Symbol
{
// Default
None,
// Control
If,
Else,
@@ -62,16 +122,63 @@ public enum Symbol
QuestionMark,
}
public abstract record Token(SourceSpan Span);
public record IdentifierToken(SourceSpan Span, string Value) : Token(Span);
public record IntLiteralToken(SourceSpan Span, string Value, int Base) : Token(Span);
public record StringLiteralToken(SourceSpan Span, string Value) : Token(Span);
public record BoolLiteralToken(SourceSpan Span, bool Value) : Token(Span);
public record FloatLiteralToken(SourceSpan Span, string Value) : Token(Span);
public record SymbolToken(SourceSpan Span, Symbol Symbol) : Token(Span);
public record SymbolToken(SourceSpan Span, Symbol Symbol) : Token(Span)
{
public override string ToString()
{
return Symbol switch
{
Symbol.Func => "func",
Symbol.If => "if",
Symbol.Else => "else",
Symbol.While => "while",
Symbol.For => "for",
Symbol.In => "in",
Symbol.Break => "break",
Symbol.Continue => "continue",
Symbol.Return => "return",
Symbol.Struct => "struct",
Symbol.Let => "let",
Symbol.Extern => "extern",
Symbol.Module => "module",
Symbol.Export => "export",
Symbol.Import => "import",
Symbol.Defer => "defer",
Symbol.Enum => "enum",
Symbol.Equal => "==",
Symbol.NotEqual => "!=",
Symbol.LessThanOrEqual => "<=",
Symbol.GreaterThanOrEqual => ">=",
Symbol.LeftShift => "<<",
Symbol.RightShift => ">>",
Symbol.And => "&&",
Symbol.Or => "||",
Symbol.DoubleColon => "::",
Symbol.Colon => ":",
Symbol.OpenParen => "(",
Symbol.CloseParen => ")",
Symbol.OpenBrace => "{",
Symbol.CloseBrace => "}",
Symbol.OpenBracket => "[",
Symbol.CloseBracket => "]",
Symbol.Comma => ",",
Symbol.Period => ".",
Symbol.Assign => "=",
Symbol.LessThan => "<",
Symbol.GreaterThan => ">",
Symbol.Plus => "+",
Symbol.Minus => "-",
Symbol.Star => "*",
Symbol.ForwardSlash => "/",
Symbol.Bang => "!",
Symbol.Caret => "^",
Symbol.Ampersand => "&",
Symbol.Semi => ";",
Symbol.Percent => "%",
Symbol.Pipe => "|",
Symbol.At => "@",
Symbol.QuestionMark => "?",
_ => "none",
};
}
}

View File

@@ -4,72 +4,9 @@ namespace NubLang.Syntax;
public sealed class Tokenizer
{
private static readonly Dictionary<string, Symbol> Keywords = new()
{
["func"] = Symbol.Func,
["if"] = Symbol.If,
["else"] = Symbol.Else,
["while"] = Symbol.While,
["for"] = Symbol.For,
["in"] = Symbol.In,
["break"] = Symbol.Break,
["continue"] = Symbol.Continue,
["return"] = Symbol.Return,
["struct"] = Symbol.Struct,
["let"] = Symbol.Let,
["extern"] = Symbol.Extern,
["module"] = Symbol.Module,
["export"] = Symbol.Export,
["import"] = Symbol.Import,
["defer"] = Symbol.Defer,
["enum"] = Symbol.Enum,
};
private static readonly Dictionary<char[], Symbol> Symbols = new()
{
[['=', '=']] = Symbol.Equal,
[['!', '=']] = Symbol.NotEqual,
[['<', '=']] = Symbol.LessThanOrEqual,
[['>', '=']] = Symbol.GreaterThanOrEqual,
[['<', '<']] = Symbol.LeftShift,
[['>', '>']] = Symbol.RightShift,
[['&', '&']] = Symbol.And,
[['|', '|']] = Symbol.Or,
[[':', ':']] = Symbol.DoubleColon,
[[':']] = Symbol.Colon,
[['(']] = Symbol.OpenParen,
[[')']] = Symbol.CloseParen,
[['{']] = Symbol.OpenBrace,
[['}']] = Symbol.CloseBrace,
[['[']] = Symbol.OpenBracket,
[[']']] = Symbol.CloseBracket,
[[',']] = Symbol.Comma,
[['.']] = Symbol.Period,
[['=']] = Symbol.Assign,
[['<']] = Symbol.LessThan,
[['>']] = Symbol.GreaterThan,
[['+']] = Symbol.Plus,
[['-']] = Symbol.Minus,
[['*']] = Symbol.Star,
[['/']] = Symbol.ForwardSlash,
[['!']] = Symbol.Bang,
[['^']] = Symbol.Caret,
[['&']] = Symbol.Ampersand,
[[';']] = Symbol.Semi,
[['%']] = Symbol.Percent,
[['|']] = Symbol.Pipe,
[['@']] = Symbol.At,
[['?']] = Symbol.QuestionMark,
};
private static readonly (char[] Pattern, Symbol Symbol)[] OrderedSymbols = Symbols
.OrderByDescending(kvp => kvp.Key.Length)
.Select(kvp => (kvp.Key, kvp.Value))
.ToArray();
private readonly string _fileName;
private readonly string _content;
private int _index = 0;
private int _index;
private int _line = 1;
private int _column = 1;
@@ -79,8 +16,8 @@ public sealed class Tokenizer
_content = content;
}
public List<Diagnostic> Diagnostics { get; } = [];
public List<Token> Tokens { get; } = [];
public List<Diagnostic> Diagnostics { get; } = new(16);
public List<Token> Tokens { get; } = new(256);
public void Tokenize()
{
@@ -90,17 +27,17 @@ public sealed class Tokenizer
_line = 1;
_column = 1;
while (Peek().HasValue)
while (_index < _content.Length)
{
try
{
var current = Peek()!.Value;
var current = _content[_index];
if (char.IsWhiteSpace(current))
{
if (current is '\n')
if (current == '\n')
{
_line += 1;
// note(nub31): Next increments the column, so 0 is correct here
_column = 0;
}
@@ -108,10 +45,10 @@ public sealed class Tokenizer
continue;
}
if (current == '/' && Peek(1) == '/')
if (current == '/' && _index + 1 < _content.Length && _content[_index + 1] == '/')
{
// note(nub31): Keep newline so next iteration increments the line counter
while (Peek() is not '\n')
Next(2);
while (_index < _content.Length && _content[_index] != '\n')
{
Next();
}
@@ -131,192 +68,312 @@ public sealed class Tokenizer
private Token ParseToken(char current, int lineStart, int columnStart)
{
if (char.IsLetter(current) || current == '_')
{
var buffer = string.Empty;
while (Peek() != null && (char.IsLetterOrDigit(Peek()!.Value) || Peek() == '_'))
{
buffer += Peek();
Next();
}
if (Keywords.TryGetValue(buffer, out var keywordSymbol))
{
return new SymbolToken(CreateSpan(lineStart, columnStart), keywordSymbol);
}
if (buffer is "true" or "false")
{
return new BoolLiteralToken(CreateSpan(lineStart, columnStart), Convert.ToBoolean(buffer));
}
return new IdentifierToken(CreateSpan(lineStart, columnStart), buffer);
}
// Numbers
if (char.IsDigit(current))
{
var buffer = string.Empty;
if (current == '0' && Peek(1) is 'x')
{
buffer += "0x";
Next();
Next();
while (Peek() != null && Uri.IsHexDigit(Peek()!.Value))
{
buffer += Peek()!.Value;
Next();
}
if (buffer.Length <= 2)
{
throw new TokenizerException(Diagnostic
.Error("Invalid hex literal, no digits found")
.At(_fileName, _line, _column)
.Build());
}
return new IntLiteralToken(CreateSpan(lineStart, columnStart), buffer, 16);
}
if (current == '0' && Peek(1) is 'b')
{
buffer += "0b";
Next();
Next();
while (Peek() != null && (Peek() == '0' || Peek() == '1'))
{
buffer += Peek()!.Value;
Next();
}
if (buffer.Length <= 2)
{
throw new TokenizerException(Diagnostic
.Error("Invalid binary literal, no digits found")
.At(_fileName, _line, _column)
.Build());
}
return new IntLiteralToken(CreateSpan(lineStart, columnStart), buffer, 2);
}
var isFloat = false;
while (Peek() != null)
{
var next = Peek()!.Value;
if (next == '.')
{
if (isFloat)
{
throw new TokenizerException(Diagnostic
.Error("More than one period found in float literal")
.At(_fileName, _line, _column)
.Build());
}
isFloat = true;
buffer += next;
Next();
}
else if (char.IsDigit(next))
{
buffer += next;
Next();
}
else
{
break;
}
}
if (isFloat)
{
return new FloatLiteralToken(CreateSpan(lineStart, columnStart), buffer);
}
else
{
return new IntLiteralToken(CreateSpan(lineStart, columnStart), buffer, 10);
}
return ParseNumber(lineStart, columnStart);
}
// String literals
if (current == '"')
{
Next();
var buffer = string.Empty;
while (true)
{
var next = Peek();
if (!next.HasValue)
{
throw new TokenizerException(Diagnostic
.Error("Unclosed string literal")
.At(_fileName, _line, _column)
.Build());
}
if (next is '\n')
{
_line += 1;
break;
}
if (next is '"')
{
Next();
break;
}
buffer += next;
Next();
}
return new StringLiteralToken(CreateSpan(lineStart, columnStart), buffer);
return ParseString(lineStart, columnStart);
}
foreach (var (pattern, symbol) in OrderedSymbols)
// Try keywords and symbols by length (longest first)
for (var i = 8; i >= 1; i--)
{
for (var i = 0; i < pattern.Length; i++)
if (TryMatchSymbol(i, lineStart, columnStart, out var token))
{
var c = Peek(i);
if (!c.HasValue || c.Value != pattern[i]) break;
if (i == pattern.Length - 1)
{
for (var j = 0; j <= i; j++)
{
Next();
}
return new SymbolToken(CreateSpan(lineStart, columnStart), symbol);
}
return token;
}
}
// Identifiers
if (char.IsLetter(current) || current == '_')
{
return ParseIdentifier(lineStart, columnStart);
}
throw new TokenizerException(Diagnostic.Error($"Unknown token '{current}'").Build());
}
private Token ParseNumber(int lineStart, int columnStart)
{
var start = _index;
var current = _content[_index];
// Hex literal
if (current == '0' && _index + 1 < _content.Length && _content[_index + 1] == 'x')
{
Next(2);
var digitStart = _index;
while (_index < _content.Length && Uri.IsHexDigit(_content[_index]))
{
Next();
}
if (_index == digitStart)
{
throw new TokenizerException(Diagnostic
.Error("Invalid hex literal, no digits found")
.At(_fileName, _line, _column)
.Build());
}
return new IntLiteralToken(
CreateSpan(lineStart, columnStart),
_content.Substring(start, _index - start),
16);
}
// Binary literal
if (current == '0' && _index + 1 < _content.Length && _content[_index + 1] == 'b')
{
Next(2);
var digitStart = _index;
while (_index < _content.Length && (_content[_index] == '0' || _content[_index] == '1'))
{
Next();
}
if (_index == digitStart)
{
throw new TokenizerException(Diagnostic
.Error("Invalid binary literal, no digits found")
.At(_fileName, _line, _column)
.Build());
}
return new IntLiteralToken(
CreateSpan(lineStart, columnStart),
_content.Substring(start, _index - start),
2);
}
// Decimal or float
var isFloat = false;
while (_index < _content.Length)
{
var next = _content[_index];
if (next == '.')
{
if (isFloat)
{
throw new TokenizerException(Diagnostic
.Error("More than one period found in float literal")
.At(_fileName, _line, _column)
.Build());
}
isFloat = true;
Next();
}
else if (char.IsDigit(next))
{
Next();
}
else
{
break;
}
}
var buffer = _content.Substring(start, _index - start);
return isFloat
? new FloatLiteralToken(CreateSpan(lineStart, columnStart), buffer)
: new IntLiteralToken(CreateSpan(lineStart, columnStart), buffer, 10);
}
private StringLiteralToken ParseString(int lineStart, int columnStart)
{
Next(); // Skip opening quote
var start = _index;
while (true)
{
if (_index >= _content.Length)
{
throw new TokenizerException(Diagnostic
.Error("Unclosed string literal")
.At(_fileName, _line, _column)
.Build());
}
var next = _content[_index];
if (next == '\n')
{
throw new TokenizerException(Diagnostic
.Error("Unclosed string literal (newline found)")
.At(_fileName, _line, _column)
.Build());
}
if (next == '"')
{
var buffer = _content.Substring(start, _index - start);
Next();
return new StringLiteralToken(CreateSpan(lineStart, columnStart), buffer);
}
Next();
}
}
private bool TryMatchSymbol(int length, int lineStart, int columnStart, out Token token)
{
token = null!;
if (_index + length > _content.Length)
{
return false;
}
var span = _content.AsSpan(_index, length);
var symbol = length switch
{
8 => span switch
{
"continue" => Symbol.Continue,
_ => Symbol.None
},
6 => span switch
{
"return" => Symbol.Return,
"struct" => Symbol.Struct,
"extern" => Symbol.Extern,
"module" => Symbol.Module,
"export" => Symbol.Export,
"import" => Symbol.Import,
_ => Symbol.None
},
5 => span switch
{
"break" => Symbol.Break,
"while" => Symbol.While,
"defer" => Symbol.Defer,
_ => Symbol.None
},
4 => span switch
{
"func" => Symbol.Func,
"else" => Symbol.Else,
"enum" => Symbol.Enum,
_ => Symbol.None
},
3 => span switch
{
"for" => Symbol.For,
"let" => Symbol.Let,
_ => Symbol.None
},
2 => span switch
{
"if" => Symbol.If,
"in" => Symbol.In,
"==" => Symbol.Equal,
"!=" => Symbol.NotEqual,
"<=" => Symbol.LessThanOrEqual,
">=" => Symbol.GreaterThanOrEqual,
"<<" => Symbol.LeftShift,
">>" => Symbol.RightShift,
"&&" => Symbol.And,
"||" => Symbol.Or,
"::" => Symbol.DoubleColon,
_ => Symbol.None
},
1 => span[0] switch
{
':' => Symbol.Colon,
'(' => Symbol.OpenParen,
')' => Symbol.CloseParen,
'{' => Symbol.OpenBrace,
'}' => Symbol.CloseBrace,
'[' => Symbol.OpenBracket,
']' => Symbol.CloseBracket,
',' => Symbol.Comma,
'.' => Symbol.Period,
'=' => Symbol.Assign,
'<' => Symbol.LessThan,
'>' => Symbol.GreaterThan,
'+' => Symbol.Plus,
'-' => Symbol.Minus,
'*' => Symbol.Star,
'/' => Symbol.ForwardSlash,
'!' => Symbol.Bang,
'^' => Symbol.Caret,
'&' => Symbol.Ampersand,
';' => Symbol.Semi,
'%' => Symbol.Percent,
'|' => Symbol.Pipe,
'@' => Symbol.At,
'?' => Symbol.QuestionMark,
_ => Symbol.None
},
_ => Symbol.None
};
if (symbol != Symbol.None)
{
var isAlphaKeyword = char.IsLetter(span[0]);
if (isAlphaKeyword)
{
var nextIdx = _index + length;
if (nextIdx < _content.Length)
{
var nextChar = _content[nextIdx];
if (char.IsLetterOrDigit(nextChar) || nextChar == '_')
{
return false;
}
}
}
Next(length);
token = new SymbolToken(CreateSpan(lineStart, columnStart), symbol);
return true;
}
return false;
}
private IdentifierToken ParseIdentifier(int lineStart, int columnStart)
{
var start = _index;
while (_index < _content.Length)
{
var ch = _content[_index];
if (char.IsLetterOrDigit(ch) || ch == '_')
{
Next();
}
else
{
break;
}
}
return new IdentifierToken(
CreateSpan(lineStart, columnStart),
_content.Substring(start, _index - start));
}
private SourceSpan CreateSpan(int lineStart, int columnStart)
{
return new SourceSpan(_fileName, new SourceLocation(lineStart, columnStart), new SourceLocation(_line, _column));
}
private char? Peek(int offset = 0)
private void Next(int count = 1)
{
if (_index + offset < _content.Length)
{
return _content[_index + offset];
}
return null;
}
private void Next()
{
_index += 1;
_column += 1;
_index += count;
_column += count;
}
}

31
compiler/NubLang/Utils.cs Normal file
View File

@@ -0,0 +1,31 @@
namespace NubLang;
public static class Utils
{
public static int LevenshteinDistance(string a, string b)
{
if (a == b) return 0;
if (a.Length == 0) return b.Length;
if (b.Length == 0) return a.Length;
var costs = new int[b.Length + 1];
for (int j = 0; j <= b.Length; j++) costs[j] = j;
for (int i = 1; i <= a.Length; i++)
{
costs[0] = i;
int prevCost = i - 1;
for (int j = 1; j <= b.Length; j++)
{
int currentCost = costs[j];
costs[j] = Math.Min(
Math.Min(costs[j - 1] + 1, costs[j] + 1),
prevCost + (a[i - 1] == b[j - 1] ? 0 : 1)
);
prevCost = currentCost;
}
}
return costs[b.Length];
}
}

View File

@@ -1,4 +1,4 @@
module "main"
module main
extern "puts" func puts(text: ^i8)

View File

@@ -1,4 +1,4 @@
module "raylib"
module raylib
export struct Vector2
{

View File

@@ -1,11 +1,9 @@
import "raylib"
import raylib
module "main"
module main
extern "main" func main(argc: i64, argv: [?]^i8): i64
{
let uwu: []i32 = [1, 2]
raylib::SetConfigFlags(raylib::ConfigFlags.FLAG_VSYNC_HINT | raylib::ConfigFlags.FLAG_WINDOW_RESIZABLE)
raylib::InitWindow(1600, 900, "Hi from nub-lang")