nub31/nub-lang
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

...

Browse Source
This commit is contained in:
nub31
2025-10-31 15:18:18 +01:00
parent 7c7624b1bc
commit 40d500fddd
8 changed files with 231 additions and 188 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
compiler/NubLang.CLI/Program.cs
View File

@@ -7,13 +7,15 @@ using NubLang.Syntax;
var diagnostics = new List<Diagnostic>();
var syntaxTrees = new List<SyntaxTree>();
var tokenizer = new Tokenizer();
var parser = new Parser();
var generator = new LlvmGenerator();
foreach (var file in args)
{
var tokenizer = new Tokenizer(file, File.ReadAllText(file));
tokenizer.Tokenize();
tokenizer.Tokenize(file, File.ReadAllText(file));
diagnostics.AddRange(tokenizer.Diagnostics);
var parser = new Parser();
var syntaxTree = parser.Parse(tokenizer.Tokens);
diagnostics.AddRange(parser.Diagnostics);
@@ -60,7 +62,6 @@ for (var i = 0; i < args.Length; i++)
var file = args[i];
var compilationUnit = compilationUnits[i];
var generator = new LlvmGenerator();
var directory = Path.GetDirectoryName(file);
if (!string.IsNullOrWhiteSpace(directory))
{

15
compiler/NubLang.LSP/DefinitionHandler.cs
View File

@@ -55,21 +55,6 @@ internal class DefinitionHandler(WorkspaceManager workspaceManager) : Definition
return null;
}
case ModuleFuncIdentifierNode funcIdentifierNode:
{
// var prototype = compilationUnit
// .ImportedFunctions
// .Where(x => x.Key.Value == funcIdentifierNode.ModuleToken.Value)
// .SelectMany(x => x.Value)
// .FirstOrDefault(x => x.NameToken.Value == funcIdentifierNode.NameToken.Value);
//
// if (prototype != null)
// {
// return new LocationOrLocationLinks(prototype.ToLocation());
// }
return null;
}
default:
{
return null;

234
compiler/NubLang.LSP/HoverHandler.cs
View File

@@ -1,5 +1,7 @@
using System.Globalization;
using NubLang.Ast;
using NubLang.Modules;
using NubLang.Types;
using OmniSharp.Extensions.LanguageServer.Protocol.Client.Capabilities;
using OmniSharp.Extensions.LanguageServer.Protocol.Document;
using OmniSharp.Extensions.LanguageServer.Protocol.Models;
@@ -29,6 +31,23 @@ internal class HoverHandler(WorkspaceManager workspaceManager) : HoverHandlerBas
return null;
}
var moduleDecl = compilationUnit.OfType<ModuleNode>().FirstOrDefault();
if (moduleDecl == null)
{
return null;
}
var moduleRepository = workspaceManager.GetModuleRepository();
if (moduleRepository == null)
{
return null;
}
if (!moduleRepository.TryGet(moduleDecl.NameToken, out var module))
{
return null;
}
var line = request.Position.Line;
var character = request.Position.Character;
@@ -39,110 +58,119 @@ internal class HoverHandler(WorkspaceManager workspaceManager) : HoverHandlerBas
return null;
}
// var message = CreateMessage(hoveredNode, compilationUnit);
// if (message == null)
// {
// return null;
// }
//
// return new Hover
// {
// Contents = new MarkedStringsOrMarkupContent(new MarkupContent
// {
// Value = message,
// Kind = MarkupKind.Markdown,
// })
// };
var message = CreateMessage(hoveredNode, moduleRepository, module);
if (message == null)
{
return null;
}
// private static string? CreateMessage(Node hoveredNode, CompilationUnit compilationUnit)
// {
// return hoveredNode switch
// {
// FuncNode funcNode => CreateFuncPrototypeMessage(funcNode.Prototype),
// FuncPrototypeNode funcPrototypeNode => CreateFuncPrototypeMessage(funcPrototypeNode),
// FuncIdentifierNode funcIdentifierNode => CreateFuncIdentifierMessage(funcIdentifierNode, compilationUnit),
// FuncParameterNode funcParameterNode => CreateTypeNameMessage("Function parameter", funcParameterNode.NameToken.Value, funcParameterNode.Type),
// VariableIdentifierNode variableIdentifierNode => CreateTypeNameMessage("Variable", variableIdentifierNode.NameToken.Value, variableIdentifierNode.Type),
// VariableDeclarationNode variableDeclarationNode => CreateTypeNameMessage("Variable declaration", variableDeclarationNode.NameToken.Value, variableDeclarationNode.Type),
// StructFieldAccessNode structFieldAccessNode => CreateTypeNameMessage("Struct field", $"{structFieldAccessNode.Target.Type}.{structFieldAccessNode.FieldToken.Value}", structFieldAccessNode.Type),
// CStringLiteralNode cStringLiteralNode => CreateLiteralMessage(cStringLiteralNode.Type, '"' + cStringLiteralNode.Value + '"'),
// StringLiteralNode stringLiteralNode => CreateLiteralMessage(stringLiteralNode.Type, '"' + stringLiteralNode.Value + '"'),
// BoolLiteralNode boolLiteralNode => CreateLiteralMessage(boolLiteralNode.Type, boolLiteralNode.Value.ToString()),
// Float32LiteralNode float32LiteralNode => CreateLiteralMessage(float32LiteralNode.Type, float32LiteralNode.Value.ToString(CultureInfo.InvariantCulture)),
// Float64LiteralNode float64LiteralNode => CreateLiteralMessage(float64LiteralNode.Type, float64LiteralNode.Value.ToString(CultureInfo.InvariantCulture)),
// I8LiteralNode i8LiteralNode => CreateLiteralMessage(i8LiteralNode.Type, i8LiteralNode.Value.ToString()),
// I16LiteralNode i16LiteralNode => CreateLiteralMessage(i16LiteralNode.Type, i16LiteralNode.Value.ToString()),
// I32LiteralNode i32LiteralNode => CreateLiteralMessage(i32LiteralNode.Type, i32LiteralNode.Value.ToString()),
// I64LiteralNode i64LiteralNode => CreateLiteralMessage(i64LiteralNode.Type, i64LiteralNode.Value.ToString()),
// U8LiteralNode u8LiteralNode => CreateLiteralMessage(u8LiteralNode.Type, u8LiteralNode.Value.ToString()),
// U16LiteralNode u16LiteralNode => CreateLiteralMessage(u16LiteralNode.Type, u16LiteralNode.Value.ToString()),
// U32LiteralNode u32LiteralNode => CreateLiteralMessage(u32LiteralNode.Type, u32LiteralNode.Value.ToString()),
// U64LiteralNode u64LiteralNode => CreateLiteralMessage(u64LiteralNode.Type, u64LiteralNode.Value.ToString()),
// // Expressions can have a generic fallback showing the resulting type
// ExpressionNode expressionNode => $"""
// **Expression** `{expressionNode.GetType().Name}`
// ```nub
// {expressionNode.Type}
// ```
// """,
// BlockNode => null,
// _ => hoveredNode.GetType().Name
// };
// }
//
// private static string CreateLiteralMessage(NubType type, string value)
// {
// return $"""
// **Literal** `{type}`
// ```nub
// {value}: {type}
// ```
// """;
// }
//
// private static string CreateTypeNameMessage(string description, string name, NubType type)
// {
// return $"""
// **{description}** `{name}`
// ```nub
// {name}: {type}
// ```
// """;
// }
//
// private static string CreateFuncIdentifierMessage(FuncIdentifierNode funcIdentifierNode, CompilationUnit compilationUnit)
// {
// var func = compilationUnit.ImportedFunctions
// .Where(x => x.Key.Value == funcIdentifierNode.ModuleToken.Value)
// .SelectMany(x => x.Value)
// .FirstOrDefault(x => x.NameToken.Value == funcIdentifierNode.NameToken.Value);
//
// if (func == null)
// {
// return $"""
// **Function** `{funcIdentifierNode.ModuleToken.Value}::{funcIdentifierNode.NameToken.Value}`
// ```nub
// // Declaration not found
// ```
// """;
// }
//
// return CreateFuncPrototypeMessage(func);
// }
//
// private static string CreateFuncPrototypeMessage(FuncPrototypeNode funcPrototypeNode)
// {
// var parameterText = string.Join(", ", funcPrototypeNode.Parameters.Select(x => $"{x.NameToken.Value}: {x.Type}"));
// var externText = funcPrototypeNode.ExternSymbolToken != null ? $"extern \"{funcPrototypeNode.ExternSymbolToken.Value}\" " : "";
//
// return $"""
// **Function** `{funcPrototypeNode.NameToken.Value}`
// ```nub
// {externText}func {funcPrototypeNode.NameToken.Value}({parameterText}): {funcPrototypeNode.ReturnType}
// ```
// """;
// }
return new Hover
{
Contents = new MarkedStringsOrMarkupContent(new MarkupContent
{
Value = message,
Kind = MarkupKind.Markdown,
})
};
}
private static string? CreateMessage(Node hoveredNode, ModuleRepository repository, ModuleRepository.Module currentModule)
{
return hoveredNode switch
{
FuncNode funcNode => CreateFuncPrototypeMessage(funcNode.Prototype),
FuncPrototypeNode funcPrototypeNode => CreateFuncPrototypeMessage(funcPrototypeNode),
LocalFuncIdentifierNode funcIdentifierNode => CreateLocalFuncIdentifierMessage(funcIdentifierNode, currentModule),
ModuleFuncIdentifierNode funcIdentifierNode => CreateModuleFuncIdentifierMessage(funcIdentifierNode, repository),
FuncParameterNode funcParameterNode => CreateTypeNameMessage("Function parameter", funcParameterNode.NameToken.Value, funcParameterNode.Type),
VariableIdentifierNode variableIdentifierNode => CreateTypeNameMessage("Variable", variableIdentifierNode.NameToken.Value, variableIdentifierNode.Type),
VariableDeclarationNode variableDeclarationNode => CreateTypeNameMessage("Variable declaration", variableDeclarationNode.NameToken.Value, variableDeclarationNode.Type),
StructFieldAccessNode structFieldAccessNode => CreateTypeNameMessage("Struct field", $"{structFieldAccessNode.Target.Type}.{structFieldAccessNode.FieldToken.Value}", structFieldAccessNode.Type),
CStringLiteralNode cStringLiteralNode => CreateLiteralMessage(cStringLiteralNode.Type, '"' + cStringLiteralNode.Value + '"'),
StringLiteralNode stringLiteralNode => CreateLiteralMessage(stringLiteralNode.Type, '"' + stringLiteralNode.Value + '"'),
BoolLiteralNode boolLiteralNode => CreateLiteralMessage(boolLiteralNode.Type, boolLiteralNode.Value.ToString()),
Float32LiteralNode float32LiteralNode => CreateLiteralMessage(float32LiteralNode.Type, float32LiteralNode.Value.ToString(CultureInfo.InvariantCulture)),
Float64LiteralNode float64LiteralNode => CreateLiteralMessage(float64LiteralNode.Type, float64LiteralNode.Value.ToString(CultureInfo.InvariantCulture)),
I8LiteralNode i8LiteralNode => CreateLiteralMessage(i8LiteralNode.Type, i8LiteralNode.Value.ToString()),
I16LiteralNode i16LiteralNode => CreateLiteralMessage(i16LiteralNode.Type, i16LiteralNode.Value.ToString()),
I32LiteralNode i32LiteralNode => CreateLiteralMessage(i32LiteralNode.Type, i32LiteralNode.Value.ToString()),
I64LiteralNode i64LiteralNode => CreateLiteralMessage(i64LiteralNode.Type, i64LiteralNode.Value.ToString()),
U8LiteralNode u8LiteralNode => CreateLiteralMessage(u8LiteralNode.Type, u8LiteralNode.Value.ToString()),
U16LiteralNode u16LiteralNode => CreateLiteralMessage(u16LiteralNode.Type, u16LiteralNode.Value.ToString()),
U32LiteralNode u32LiteralNode => CreateLiteralMessage(u32LiteralNode.Type, u32LiteralNode.Value.ToString()),
U64LiteralNode u64LiteralNode => CreateLiteralMessage(u64LiteralNode.Type, u64LiteralNode.Value.ToString()),
// Expressions can have a generic fallback showing the resulting type
ExpressionNode expressionNode => $"""
**Expression** `{expressionNode.GetType().Name}`
```nub
{expressionNode.Type}
```
""",
BlockNode => null,
_ => hoveredNode.GetType().Name
};
}
private static string CreateLiteralMessage(NubType type, string value)
{
return $"""
**Literal** `{type}`
```nub
{value}: {type}
```
""";
}
private static string CreateTypeNameMessage(string description, string name, NubType type)
{
return $"""
**{description}** `{name}`
```nub
{name}: {type}
```
""";
}
private static string CreateLocalFuncIdentifierMessage(LocalFuncIdentifierNode funcIdentifierNode, ModuleRepository.Module currentModule)
{
if (!currentModule.TryResolveFunc(funcIdentifierNode.NameToken, out var func, out _))
{
return $"""
**Function** `{funcIdentifierNode.NameToken.Value}`
```nub
// Declaration not found
```
""";
}
return CreateFuncPrototypeMessage(func);
}
private static string CreateModuleFuncIdentifierMessage(ModuleFuncIdentifierNode funcIdentifierNode, ModuleRepository repository)
{
if (!repository.TryGet(funcIdentifierNode.ModuleToken, out var module) || !module.TryResolveFunc(funcIdentifierNode.NameToken, out var func, out _))
{
return $"""
**Function** `{funcIdentifierNode.ModuleToken.Value}::{funcIdentifierNode.NameToken.Value}`
```nub
// Declaration not found
```
""";
}
return CreateFuncPrototypeMessage(func);
}
private static string CreateFuncPrototypeMessage(FuncPrototypeNode funcPrototypeNode)
{
var parameterText = string.Join(", ", funcPrototypeNode.Parameters.Select(x => $"{x.NameToken.Value}: {x.Type}"));
var externText = funcPrototypeNode.ExternSymbolToken != null ? $"extern \"{funcPrototypeNode.ExternSymbolToken.Value}\" " : "";
return $"""
**Function** `{funcPrototypeNode.NameToken.Value}`
```nub
{externText}func {funcPrototypeNode.NameToken.Value}({parameterText}): {funcPrototypeNode.ReturnType}
```
""";
}
}

126
compiler/NubLang.LSP/WorkspaceManager.cs
View File

@@ -1,4 +1,6 @@
using NubLang.Ast;
using NubLang.Diagnostics;
using NubLang.Modules;
using NubLang.Syntax;
using OmniSharp.Extensions.LanguageServer.Protocol;
@@ -8,70 +10,92 @@ public class WorkspaceManager(DiagnosticsPublisher diagnosticsPublisher)
{
private readonly Dictionary<string, SyntaxTree> _syntaxTrees = new();
private readonly Dictionary<string, List<TopLevelNode>> _compilationUnits = new();
// private readonly Dictionary<string, TypedModule> _modules = new();
private ModuleRepository? _repository;
public void Init(string rootPath)
{
// var files = Directory.GetFiles(rootPath, "*.nub", SearchOption.AllDirectories);
// foreach (var path in files)
// {
// var text = File.ReadAllText(path);
// var tokenizer = new Tokenizer(path, text);
//
// tokenizer.Tokenize();
// diagnosticsPublisher.Publish(path, tokenizer.Diagnostics);
//
// var parser = new Parser();
// var parseResult = parser.Parse(tokenizer.Tokens);
// diagnosticsPublisher.Publish(path, parser.Diagnostics);
//
// _syntaxTrees[path] = parseResult;
// }
//
// foreach (var (fsPath, syntaxTree) in _syntaxTrees)
// {
// var modules = Module.Collect(_syntaxTrees.Select(x => x.Value).ToList());
//
// var typeChecker = new TypeChecker(syntaxTree, modules);
// var result = typeChecker.Check();
// diagnosticsPublisher.Publish(fsPath, typeChecker.Diagnostics);
//
// _compilationUnits[fsPath] = result;
// }
var files = Directory.GetFiles(rootPath, "*.nub", SearchOption.AllDirectories);
foreach (var path in files)
{
var text = File.ReadAllText(path);
var tokenizer = new Tokenizer();
tokenizer.Tokenize(path, text);
diagnosticsPublisher.Publish(path, tokenizer.Diagnostics);
var parser = new Parser();
var parseResult = parser.Parse(tokenizer.Tokens);
diagnosticsPublisher.Publish(path, parser.Diagnostics);
_syntaxTrees[path] = parseResult;
}
ModuleRepository repository;
foreach (var (fsPath, syntaxTree) in _syntaxTrees)
{
try
{
repository = ModuleRepository.Create(_syntaxTrees.Select(x => x.Value).ToList());
}
catch (CompileException e)
{
return;
}
var typeChecker = new TypeChecker();
var result = typeChecker.Check(syntaxTree, repository);
diagnosticsPublisher.Publish(fsPath, typeChecker.Diagnostics);
_compilationUnits[fsPath] = result;
}
}
public void UpdateFile(DocumentUri path)
{
// var fsPath = path.GetFileSystemPath();
//
// var text = File.ReadAllText(fsPath);
// var tokenizer = new Tokenizer(fsPath, text);
// tokenizer.Tokenize();
// diagnosticsPublisher.Publish(path, tokenizer.Diagnostics);
//
// var parser = new Parser();
// var syntaxTree = parser.Parse(tokenizer.Tokens);
// diagnosticsPublisher.Publish(path, parser.Diagnostics);
// _syntaxTrees[fsPath] = syntaxTree;
//
// var modules = Module.Collect(_syntaxTrees.Select(x => x.Value).ToList());
//
// var typeChecker = new TypeChecker(syntaxTree, modules);
// var result = typeChecker.Check();
// diagnosticsPublisher.Publish(fsPath, typeChecker.Diagnostics);
//
// _compilationUnits[fsPath] = result;
var fsPath = path.GetFileSystemPath();
var text = File.ReadAllText(fsPath);
var tokenizer = new Tokenizer();
tokenizer.Tokenize(fsPath, text);
diagnosticsPublisher.Publish(path, tokenizer.Diagnostics);
var parser = new Parser();
var syntaxTree = parser.Parse(tokenizer.Tokens);
diagnosticsPublisher.Publish(path, parser.Diagnostics);
_syntaxTrees[fsPath] = syntaxTree;
ModuleRepository repository;
try
{
repository = ModuleRepository.Create(_syntaxTrees.Select(x => x.Value).ToList());
}
catch (CompileException e)
{
diagnosticsPublisher.Publish(path, [e.Diagnostic]);
return;
}
public void RemoveFile(DocumentUri path)
{
var fsPath = path.GetFileSystemPath();
_syntaxTrees.Remove(fsPath);
_compilationUnits.Remove(fsPath);
var typeChecker = new TypeChecker();
var result = typeChecker.Check(syntaxTree, repository);
diagnosticsPublisher.Publish(fsPath, typeChecker.Diagnostics);
_compilationUnits[fsPath] = result;
}
public List<TopLevelNode>? GetCompilationUnit(DocumentUri path)
{
return _compilationUnits.GetValueOrDefault(path.GetFileSystemPath());
}
public ModuleRepository? GetModuleRepository()
{
try
{
return ModuleRepository.Create(_syntaxTrees.Select(x => x.Value).ToList());
}
catch (CompileException e)
{
return null;
}
}
}

4
compiler/NubLang/Diagnostics/Diagnostic.cs
View File

@@ -105,8 +105,8 @@ public class Diagnostic
sb.AppendLine();
var text = File.ReadAllText(Span.Value.FilePath);
var tokenizer = new Tokenizer(Span.Value.FilePath, text);
tokenizer.Tokenize();
var tokenizer = new Tokenizer();
tokenizer.Tokenize(Span.Value.FilePath, text);
var lines = text.Split('\n');

6
compiler/NubLang/Modules/ModuleRepository.cs
View File

@@ -161,6 +161,12 @@ public sealed class ModuleRepository
return module;
}
public bool TryGet(IdentifierToken ident, [NotNullWhen(true)] out Module? module)
{
module = _modules.GetValueOrDefault(ident.Value);
return module != null;
}
public sealed class Module
{
public required string Name { get; init; }

5
compiler/NubLang/Syntax/Parser.cs
View File

@@ -11,14 +11,15 @@ public sealed class Parser
private Token? CurrentToken => _tokenIndex < _tokens.Count ? _tokens[_tokenIndex] : null;
private bool HasToken => CurrentToken != null;
public List<Diagnostic> Diagnostics { get; } = [];
public List<Diagnostic> Diagnostics { get; set; } = [];
public SyntaxTree Parse(List<Token> tokens)
{
Diagnostics.Clear();
_tokens = tokens;
_tokenIndex = 0;
Diagnostics = [];
var topLevelSyntaxNodes = new List<TopLevelSyntaxNode>();
while (HasToken)

18
compiler/NubLang/Syntax/Tokenizer.cs
View File

@@ -4,25 +4,23 @@ namespace NubLang.Syntax;
public sealed class Tokenizer
{
private readonly string _fileName;
private readonly string _content;
private string _fileName = null!;
private string _content = null!;
private int _index;
private int _line = 1;
private int _column = 1;
public Tokenizer(string fileName, string content)
public List<Diagnostic> Diagnostics { get; set; } = new(16);
public List<Token> Tokens { get; set; } = new(256);
public void Tokenize(string fileName, string content)
{
_fileName = fileName;
_content = content;
}
public List<Diagnostic> Diagnostics { get; } = new(16);
public List<Token> Tokens { get; } = new(256);
Diagnostics = [];
Tokens = [];
public void Tokenize()
{
Diagnostics.Clear();
Tokens.Clear();
_index = 0;
_line = 1;
_column = 1;