nub31/nub-lang-archive
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

...

Browse Source
This commit is contained in:
nub31
2025-05-24 20:37:26 +02:00
parent 010df49dc4
commit 757e3490f5
9 changed files with 132 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
example/program.nub
View File

@@ -1,9 +1,14 @@
import c
// Test
// Test2
// Test3
// Test4
global func main(args: []string) {
i = 0
printf("%d\n", args.count)
while i < args.count {
// Test
printf("%s\n", args[i])
i = i + 1
}

6
src/compiler/Nub.Lang/Frontend/Lexing/CommentToken.cs Normal file
View File

@@ -0,0 +1,6 @@
namespace Nub.Lang.Frontend.Lexing;
public class CommentToken(string comment) : Token
{
public string Comment { get; } = comment;
}

119
src/compiler/Nub.Lang/Frontend/Lexing/Lexer.cs
View File

@@ -3,7 +3,7 @@
public class Lexer
{
private static readonly Dictionary<string, Symbol> Keywords = new()
{
{
["func"] = Symbol.Func,
["import"] = Symbol.Import,
["if"] = Symbol.If,
@@ -15,9 +15,9 @@ public class Lexer
["new"] = Symbol.New,
["struct"] = Symbol.Struct,
};
private static readonly Dictionary<string, Modifier> Modifers = new()
{
{
["global"] = Modifier.Global,
["extern"] = Modifier.Extern,
};
@@ -29,7 +29,7 @@ public class Lexer
[['<', '=']] = Symbol.LessThanOrEqual,
[['>', '=']] = Symbol.GreaterThanOrEqual,
};
private static readonly Dictionary<char, Symbol> Chars = new()
{
[';'] = Symbol.Semicolon,
@@ -53,7 +53,7 @@ public class Lexer
['^'] = Symbol.Caret,
['&'] = Symbol.Ampersand,
};
private string _src = string.Empty;
private int _index;
@@ -61,40 +61,58 @@ public class Lexer
{
_src = src;
_index = 0;
List<Token> tokens = [];
while (Peek().HasValue)
while (Peek().TryGetValue(out var character))
{
tokens.Add(ParseToken());
if (char.IsWhiteSpace(character))
{
Next();
continue;
}
tokens.Add(ParseToken(character));
}
return tokens;
}
private Token ParseToken()
private Token ParseToken(char current)
{
var current = Peek();
if (char.IsLetter(current.Value) || current.Value == '_')
if (current == '/' && Peek(1) is { Value: '/' })
{
Next();
Next();
var buffer = string.Empty;
while (Peek() is not { Value: '\n' })
{
buffer += Peek().Value;
Next();
}
Next();
return new CommentToken(buffer);
}
if (char.IsLetter(current) || current == '_')
{
var buffer = string.Empty;
while (current.HasValue && (char.IsLetterOrDigit(current.Value) || current.Value == '_'))
while (Peek().TryGetValue(out var next) && (char.IsLetterOrDigit(next) || next == '_'))
{
buffer += current.Value;
buffer += next;
Next();
current = Peek();
}
if (Keywords.TryGetValue(buffer, out var keywordSymbol))
{
return new SymbolToken(keywordSymbol);
}
if (Modifers.TryGetValue(buffer, out var modifer))
{
return new ModifierToken(modifer);
}
if (buffer is "true" or "false")
{
return new LiteralToken(NubPrimitiveType.Bool, buffer);
@@ -103,31 +121,30 @@ public class Lexer
return new IdentifierToken(buffer);
}
if (char.IsDigit(current.Value))
if (char.IsDigit(current))
{
var isFloat = false;
var buffer = string.Empty;
while (current.HasValue)
while (Peek().TryGetValue(out var next))
{
if (current.Value == '.')
if (next == '.')
{
if (isFloat)
{
throw new Exception("More than one period found in float literal");
}
isFloat = true;
buffer += current.Value;
buffer += next;
Next();
current = Peek();
}
else if (char.IsDigit(current.Value))
{
buffer += current.Value;
Next();
current = Peek();
}
else if (current.Value == 'f')
else if (char.IsDigit(next))
{
buffer += next;
Next();
}
else if (next == 'f')
{
isFloat = true;
Next();
@@ -138,15 +155,15 @@ public class Lexer
break;
}
}
return new LiteralToken(isFloat ? NubPrimitiveType.F64 : NubPrimitiveType.I64, buffer);
}
// TODO: Revisit this
foreach (var chain in Chians)
{
if (current.Value != chain.Key[0]) continue;
if (current != chain.Key[0]) continue;
for (var i = 1; i < chain.Key.Length; i++)
{
var c = Peek(i);
@@ -164,36 +181,38 @@ public class Lexer
}
}
if (Chars.TryGetValue(current.Value, out var charSymbol))
if (Chars.TryGetValue(current, out var charSymbol))
{
Next();
return new SymbolToken(charSymbol);
}
if (current.Value == '"')
if (current == '"')
{
Next();
var buffer = string.Empty;
while (true)
{
current = Peek();
if (!Peek().TryGetValue(out var next))
{
throw new Exception("Unclosed string literal");
}
if (next == '"')
{
Next();
break;
}
buffer += next;
Next();
if (!current.HasValue) throw new Exception("Unclosed string literal");
if (current.Value == '"') break;
buffer += current.Value;
}
return new LiteralToken(NubPrimitiveType.String, buffer);
}
if (char.IsWhiteSpace(current.Value))
{
Next();
return new SymbolToken(Symbol.Whitespace);
}
throw new Exception($"Unknown character {current.Value}");
throw new Exception($"Unknown character {current}");
}
private Optional<char> Peek(int offset = 0)
@@ -202,7 +221,7 @@ public class Lexer
{
return _src[_index + offset];
}
return Optional<char>.Empty();
}

1
src/compiler/Nub.Lang/Frontend/Lexing/SymbolToken.cs
View File

@@ -7,7 +7,6 @@ public class SymbolToken(Symbol symbol) : Token
public enum Symbol
{
Whitespace,
Import,
Func,
Return,

3
src/compiler/Nub.Lang/Frontend/Parsing/ExternFuncDefinitionNode.cs
View File

@@ -1,10 +1,11 @@
namespace Nub.Lang.Frontend.Parsing;
public class ExternFuncDefinitionNode(string name, List<FuncParameter> parameters, Optional<NubType> returnType) : DefinitionNode
public class ExternFuncDefinitionNode(string name, List<FuncParameter> parameters, Optional<NubType> returnType, Optional<string> documentation) : DefinitionNode
{
public string Name { get; } = name;
public List<FuncParameter> Parameters { get; } = parameters;
public Optional<NubType> ReturnType { get; } = returnType;
public Optional<string> Documentation { get; set; } = documentation;
public override string ToString() => $"{Name}({string.Join(", ", Parameters.Select(p => p.ToString()))}){(ReturnType.HasValue ? ": " + ReturnType.Value : "")}";
}

3
src/compiler/Nub.Lang/Frontend/Parsing/LocalFuncDefinitionNode.cs
View File

@@ -1,12 +1,13 @@
namespace Nub.Lang.Frontend.Parsing;
public class LocalFuncDefinitionNode(string name, List<FuncParameter> parameters, BlockNode body, Optional<NubType> returnType, bool global) : DefinitionNode
public class LocalFuncDefinitionNode(string name, List<FuncParameter> parameters, BlockNode body, Optional<NubType> returnType, bool global, Optional<string> documentation) : DefinitionNode
{
public string Name { get; } = name;
public List<FuncParameter> Parameters { get; } = parameters;
public BlockNode Body { get; } = body;
public Optional<NubType> ReturnType { get; } = returnType;
public bool Global { get; } = global;
public Optional<string> Documentation { get; set; } = documentation;
public override string ToString() => $"{Name}({string.Join(", ", Parameters.Select(p => p.ToString()))}){(ReturnType.HasValue ? ": " + ReturnType.Value : "")}";
}

47
src/compiler/Nub.Lang/Frontend/Parsing/Parser.cs
View File

@@ -36,6 +36,15 @@ public class Parser
{
List<Modifier> modifiers = [];
List<string> documentationParts = [];
while (_index < _tokens.Count && _tokens[_index] is CommentToken commentToken)
{
documentationParts.Add(commentToken.Comment);
_index++;
}
var documentation = documentationParts.Count == 0 ? null : string.Join('\n', documentationParts);
while (TryExpectModifier(out var modifier))
{
modifiers.Add(modifier);
@@ -44,13 +53,13 @@ public class Parser
var keyword = ExpectSymbol();
return keyword.Symbol switch
{
Symbol.Func => ParseFuncDefinition(modifiers),
Symbol.Struct => ParseStruct(modifiers),
Symbol.Func => ParseFuncDefinition(modifiers, Optional.OfNullable(documentation)),
Symbol.Struct => ParseStruct(modifiers, Optional.OfNullable(documentation)),
_ => throw new Exception("Unexpected symbol: " + keyword.Symbol)
};
}
private DefinitionNode ParseFuncDefinition(List<Modifier> modifiers)
private DefinitionNode ParseFuncDefinition(List<Modifier> modifiers, Optional<string> documentation)
{
var name = ExpectIdentifier();
List<FuncParameter> parameters = [];
@@ -77,7 +86,7 @@ public class Parser
throw new Exception($"Modifiers: {string.Join(", ", modifiers)} is not valid for an extern function");
}
return new ExternFuncDefinitionNode(name.Value, parameters, returnType);
return new ExternFuncDefinitionNode(name.Value, parameters, returnType, documentation);
}
var body = ParseBlock();
@@ -88,10 +97,10 @@ public class Parser
throw new Exception($"Modifiers: {string.Join(", ", modifiers)} is not valid for a local function");
}
return new LocalFuncDefinitionNode(name.Value, parameters, body, returnType, global);
return new LocalFuncDefinitionNode(name.Value, parameters, body, returnType, global, documentation);
}
private StructDefinitionNode ParseStruct(List<Modifier> _)
private StructDefinitionNode ParseStruct(List<Modifier> _, Optional<string> documentation)
{
var name = ExpectIdentifier().Value;
@@ -115,7 +124,7 @@ public class Parser
variables.Add(new StructField(variableName, variableType, variableValue));
}
return new StructDefinitionNode(name, variables);
return new StructDefinitionNode(name, variables, documentation);
}
private FuncParameter ParseFuncParameter()
@@ -230,7 +239,7 @@ public class Parser
private ExpressionNode ParseExpression(int precedence = 0)
{
var left = ParsePrimaryExpression();
while (true)
{
var token = Peek();
@@ -308,7 +317,7 @@ public class Parser
private ExpressionNode ParsePrimaryExpression()
{
ExpressionNode expr;
var token = ExpectToken();
switch (token)
{
@@ -341,6 +350,7 @@ public class Parser
break;
}
}
break;
}
case SymbolToken symbolToken:
@@ -403,6 +413,7 @@ public class Parser
throw new Exception($"Unknown symbol: {symbolToken.Symbol}");
}
}
break;
}
default:
@@ -463,7 +474,7 @@ public class Parser
{
return NubType.Parse(name);
}
if (TryExpectSymbol(Symbol.Caret))
{
var baseType = ParseType();
@@ -558,14 +569,15 @@ public class Parser
private Optional<Token> Peek()
{
while (_index < _tokens.Count && _tokens.ElementAt(_index) is SymbolToken { Symbol: Symbol.Whitespace })
var peekIndex = _index;
while (peekIndex < _tokens.Count && _tokens[peekIndex] is CommentToken)
{
Next();
peekIndex++;
}
if (_index < _tokens.Count)
if (peekIndex < _tokens.Count)
{
return _tokens.ElementAt(_index);
return _tokens[peekIndex];
}
return Optional<Token>.Empty();
@@ -573,6 +585,11 @@ public class Parser
private void Next()
{
while (_index < _tokens.Count && _tokens[_index] is CommentToken)
{
_index++;
}
_index++;
}
}

3
src/compiler/Nub.Lang/Frontend/Parsing/StructDefinitionNode.cs
View File

@@ -1,7 +1,8 @@
namespace Nub.Lang.Frontend.Parsing;
public class StructDefinitionNode(string name, List<StructField> fields) : DefinitionNode
public class StructDefinitionNode(string name, List<StructField> fields, Optional<string> documentation) : DefinitionNode
{
public string Name { get; } = name;
public List<StructField> Fields { get; } = fields;
public Optional<string> Documentation { get; set; } = documentation;
}

14
src/compiler/Nub.Lang/Optional.cs
View File

@@ -44,5 +44,19 @@ public readonly struct Optional<TValue>
[MemberNotNullWhen(true, nameof(Value))]
public bool HasValue { get; }
[MemberNotNullWhen(true, nameof(Value))]
public bool TryGetValue([NotNullWhen(true)] out TValue? value)
{
if (HasValue)
{
value = Value;
return true;
}
value = default;
return false;
}
public static implicit operator Optional<TValue>(TValue value) => new(value);
}