WIP: dev #1

Draft
nub31 wants to merge 103 commits from dev into master
4 changed files with 208 additions and 32 deletions
Showing only changes of commit e20f6cd7af - Show all commits

View File

@@ -0,0 +1,77 @@
namespace Compiler;
public sealed class Diagnostic(DiagnosticSeverity severity, string message, string? help, FileInfo? file)
{
public static DiagnosticBuilder Info(string message) => new DiagnosticBuilder(DiagnosticSeverity.Info, message);
public static DiagnosticBuilder Warning(string message) => new DiagnosticBuilder(DiagnosticSeverity.Warning, message);
public static DiagnosticBuilder Error(string message) => new DiagnosticBuilder(DiagnosticSeverity.Error, message);
public readonly DiagnosticSeverity Severity = severity;
public readonly string Message = message;
public readonly string? Help = help;
public readonly FileInfo? File = file;
}
public sealed class DiagnosticBuilder(DiagnosticSeverity severity, string message)
{
private FileInfo? file;
private string? help;
public DiagnosticBuilder At(string fileName, int line, int column, int length)
{
file = new FileInfo(fileName, line, column, length);
return this;
}
public DiagnosticBuilder At(string fileName, Token? token)
{
if (token != null)
{
At(fileName, token.Line, token.Column, token.Length);
}
return this;
}
public DiagnosticBuilder At(string fileName, Node? node)
{
if (node != null && node.Tokens.Count != 0)
{
// todo(nub31): Calculate length based on last token
At(fileName, node.Tokens[0]);
}
return this;
}
public DiagnosticBuilder WithHelp(string helpMessage)
{
help = helpMessage;
return this;
}
public Diagnostic Build()
{
return new Diagnostic(severity, message, help, file);
}
}
public sealed class FileInfo(string file, int line, int column, int length)
{
public readonly string File = file;
public readonly int Line = line;
public readonly int Column = column;
public readonly int Length = length;
}
public enum DiagnosticSeverity
{
Info,
Warning,
Error,
}
public sealed class CompileException(Diagnostic diagnostic) : Exception
{
public readonly Diagnostic Diagnostic = diagnostic;
}

View File

@@ -2,21 +2,31 @@
namespace Compiler;
public sealed class Parser(List<Token> tokens)
public sealed class Parser(string fileName, List<Token> tokens)
{
public static List<NodeDefinition> Parse(List<Token> tokens)
public static List<NodeDefinition> Parse(string fileName, List<Token> tokens, out List<Diagnostic> diagnostics)
{
return new Parser(tokens).Parse();
return new Parser(fileName, tokens).Parse(out diagnostics);
}
private int index;
private List<NodeDefinition> Parse()
private List<NodeDefinition> Parse(out List<Diagnostic> diagnostics)
{
var nodes = new List<NodeDefinition>();
diagnostics = [];
while (Peek() != null)
nodes.Add(ParseDefinition());
try
{
while (Peek() != null)
{
nodes.Add(ParseDefinition());
}
}
catch (CompileException e)
{
diagnostics.Add(e.Diagnostic);
}
return nodes;
}
@@ -66,7 +76,7 @@ public sealed class Parser(List<Token> tokens)
return new NodeDefinitionStruct(TokensFrom(startIndex), name, fields);
}
throw new Exception("Not a valid definition");
throw new CompileException(Diagnostic.Error("Not a valid definition").At(fileName, Peek()).Build());
}
private NodeStatement ParseStatement()
@@ -135,7 +145,7 @@ public sealed class Parser(List<Token> tokens)
return new NodeStatementAssignment(TokensFrom(startIndex), target, value);
}
throw new Exception("Not a valid followup for expression statement");
throw new CompileException(Diagnostic.Error("Cannot use expression in statement context unless called as a function or used in assignment").At(fileName, target).Build());
}
private NodeExpression ParseExpression(int minPrecedence = -1)
@@ -233,7 +243,7 @@ public sealed class Parser(List<Token> tokens)
}
else
{
throw new Exception("Not a valid expression leaf");
throw new CompileException(Diagnostic.Error("Expected start of expression").At(fileName, Peek()).Build());
}
if (TryExpectSymbol(Symbol.Period))
@@ -302,7 +312,7 @@ public sealed class Parser(List<Token> tokens)
}
}
throw new Exception("Not a valid type");
throw new CompileException(Diagnostic.Error("Expected type").At(fileName, Peek()).Build());
}
private List<Token> TokensFrom(int startIndex)
@@ -329,7 +339,7 @@ public sealed class Parser(List<Token> tokens)
return;
}
throw new Exception($"Expected symbol '{symbol}'");
throw new CompileException(Diagnostic.Error($"Expected '{symbol.AsString()}'").At(fileName, Peek()).Build());
}
private bool TryExpectSymbol(Symbol symbol)
@@ -351,7 +361,7 @@ public sealed class Parser(List<Token> tokens)
return token;
}
throw new Exception("Expected ident");
throw new CompileException(Diagnostic.Error("Expected identifier").At(fileName, Peek()).Build());
}
private bool TryExpectIdent([NotNullWhen(true)] out TokenIdent? ident)
@@ -409,7 +419,7 @@ public sealed class Parser(List<Token> tokens)
private Token Consume()
{
if (index >= tokens.Count)
throw new Exception("End of tokens");
throw new CompileException(Diagnostic.Error("Unexpected end of tokens").At(fileName, Peek()).Build());
return tokens[index++];
}

View File

@@ -6,7 +6,7 @@ const string contents = """
name: string
}
func main(): i32 {
func main(: i32 {
let x: i32 = 23
x = 24
@@ -35,8 +35,32 @@ const string contents = """
}
""";
var tokens = Tokenizer.Tokenize(contents);
var nodes = Parser.Parse(tokens);
var tokens = Tokenizer.Tokenize("test.nub", contents, out var tokenizerDiagnostics);
foreach (var diagnostic in tokenizerDiagnostics)
{
Console.WriteLine(diagnostic.Message);
}
if (tokenizerDiagnostics.Any(x => x.Severity == DiagnosticSeverity.Error))
{
return 1;
}
var nodes = Parser.Parse("test.nub", tokens, out var parserDiagnostics);
foreach (var diagnostic in parserDiagnostics)
{
Console.WriteLine(diagnostic.Message);
}
if (parserDiagnostics.Any(x => x.Severity == DiagnosticSeverity.Error))
{
return 1;
}
var output = Generator.Emit(nodes);
File.WriteAllText("C:/Users/oliste/repos/nub-lang/compiler/Compiler/out.c", output);
File.WriteAllText("C:/Users/oliste/repos/nub-lang/compiler/Compiler/out.c", output);
return 0;

View File

@@ -3,33 +3,40 @@ using System.Text;
namespace Compiler;
public sealed class Tokenizer(string contents)
public sealed class Tokenizer(string fileName, string contents)
{
public static List<Token> Tokenize(string contents)
public static List<Token> Tokenize(string fileName, string contents, out List<Diagnostic> diagnostics)
{
return new Tokenizer(contents).Tokenize();
return new Tokenizer(fileName, contents).Tokenize(out diagnostics);
}
private int index;
private int line = 1;
private int column = 1;
private List<Token> Tokenize()
private List<Token> Tokenize(out List<Diagnostic> diagnostics)
{
var tokens = new List<Token>();
while (true)
diagnostics = [];
try
{
if (!TryPeek(out var c))
break;
if (char.IsWhiteSpace(c))
while (true)
{
Consume();
continue;
}
if (!TryPeek(out var c))
break;
tokens.Add(ParseToken());
if (char.IsWhiteSpace(c))
{
Consume();
continue;
}
tokens.Add(ParseToken());
}
}
catch (CompileException e)
{
diagnostics.Add(e.Diagnostic);
}
return tokens;
@@ -347,7 +354,7 @@ public sealed class Tokenizer(string contents)
private char Consume()
{
if (index >= contents.Length)
throw new Exception("End of tokens");
throw new CompileException(Diagnostic.Error("Unexpected end of file").At(fileName, line, column, 0).Build());
var c = contents[index];
@@ -469,4 +476,62 @@ public enum Keyword
public sealed class TokenKeyword(int line, int column, int length, Keyword keyword) : Token(line, column, length)
{
public readonly Keyword Keyword = keyword;
}
public static class TokenExtensions
{
public static string AsString(this Symbol symbol)
{
return symbol switch
{
Symbol.OpenCurly => "{",
Symbol.CloseCurly => "}",
Symbol.OpenParen => "(",
Symbol.CloseParen => ")",
Symbol.Comma => ",",
Symbol.Period => ",",
Symbol.Colon => ":",
Symbol.Caret => "^",
Symbol.Bang => "!",
Symbol.Equal => "=",
Symbol.EqualEqual => "==",
Symbol.BangEqual => "!+",
Symbol.LessThan => "<",
Symbol.LessThanLessThan => "<<",
Symbol.LessThanEqual => "<=",
Symbol.GreaterThan => ">",
Symbol.GreaterThanGreaterThan => ">>",
Symbol.GreaterThanEqual => ">=",
Symbol.Plus => "+",
Symbol.PlusEqual => "+=",
Symbol.Minus => "-",
Symbol.MinusEqual => "-=",
Symbol.Star => "*",
Symbol.StarEqual => "*=",
Symbol.ForwardSlash => "/",
Symbol.ForwardSlashEqual => "/=",
Symbol.Percent => "%",
Symbol.PercentEqual => "%=",
Symbol.Ampersand => "&",
Symbol.AmpersandAmpersand => "&&",
Symbol.Pipe => "|",
Symbol.PipePipe => "||",
_ => throw new ArgumentOutOfRangeException(nameof(symbol), symbol, null)
};
}
public static string AsString(this Keyword symbol)
{
return symbol switch
{
Keyword.Func => "func",
Keyword.Struct => "struct",
Keyword.Let => "let",
Keyword.If => "if",
Keyword.Else => "else",
Keyword.While => "while",
Keyword.Return => "return",
_ => throw new ArgumentOutOfRangeException(nameof(symbol), symbol, null)
};
}
}