diff --git a/example/program.nub b/example/program.nub index e1cfe83..d78d772 100644 --- a/example/program.nub +++ b/example/program.nub @@ -1,9 +1,14 @@ import c +// Test +// Test2 +// Test3 +// Test4 global func main(args: []string) { i = 0 printf("%d\n", args.count) while i < args.count { + // Test printf("%s\n", args[i]) i = i + 1 } diff --git a/src/compiler/Nub.Lang/Frontend/Lexing/CommentToken.cs b/src/compiler/Nub.Lang/Frontend/Lexing/CommentToken.cs new file mode 100644 index 0000000..f4568f1 --- /dev/null +++ b/src/compiler/Nub.Lang/Frontend/Lexing/CommentToken.cs @@ -0,0 +1,6 @@ +namespace Nub.Lang.Frontend.Lexing; + +public class CommentToken(string comment) : Token +{ + public string Comment { get; } = comment; +} \ No newline at end of file diff --git a/src/compiler/Nub.Lang/Frontend/Lexing/Lexer.cs b/src/compiler/Nub.Lang/Frontend/Lexing/Lexer.cs index 5255371..0a41049 100644 --- a/src/compiler/Nub.Lang/Frontend/Lexing/Lexer.cs +++ b/src/compiler/Nub.Lang/Frontend/Lexing/Lexer.cs @@ -3,7 +3,7 @@ public class Lexer { private static readonly Dictionary Keywords = new() - { + { ["func"] = Symbol.Func, ["import"] = Symbol.Import, ["if"] = Symbol.If, @@ -15,9 +15,9 @@ public class Lexer ["new"] = Symbol.New, ["struct"] = Symbol.Struct, }; - + private static readonly Dictionary Modifers = new() - { + { ["global"] = Modifier.Global, ["extern"] = Modifier.Extern, }; @@ -29,7 +29,7 @@ public class Lexer [['<', '=']] = Symbol.LessThanOrEqual, [['>', '=']] = Symbol.GreaterThanOrEqual, }; - + private static readonly Dictionary Chars = new() { [';'] = Symbol.Semicolon, @@ -53,7 +53,7 @@ public class Lexer ['^'] = Symbol.Caret, ['&'] = Symbol.Ampersand, }; - + private string _src = string.Empty; private int _index; @@ -61,40 +61,58 @@ public class Lexer { _src = src; _index = 0; - + List tokens = []; - while (Peek().HasValue) + while (Peek().TryGetValue(out var character)) { - tokens.Add(ParseToken()); + if (char.IsWhiteSpace(character)) + { + Next(); + continue; + } + tokens.Add(ParseToken(character)); } + return tokens; } - private Token ParseToken() + private Token ParseToken(char current) { - var current = Peek(); - - if (char.IsLetter(current.Value) || current.Value == '_') + if (current == '/' && Peek(1) is { Value: '/' }) + { + Next(); + Next(); + var buffer = string.Empty; + while (Peek() is not { Value: '\n' }) + { + buffer += Peek().Value; + Next(); + } + + Next(); + return new CommentToken(buffer); + } + + if (char.IsLetter(current) || current == '_') { var buffer = string.Empty; - - while (current.HasValue && (char.IsLetterOrDigit(current.Value) || current.Value == '_')) + + while (Peek().TryGetValue(out var next) && (char.IsLetterOrDigit(next) || next == '_')) { - buffer += current.Value; + buffer += next; Next(); - current = Peek(); } if (Keywords.TryGetValue(buffer, out var keywordSymbol)) { return new SymbolToken(keywordSymbol); } - + if (Modifers.TryGetValue(buffer, out var modifer)) { return new ModifierToken(modifer); } - + if (buffer is "true" or "false") { return new LiteralToken(NubPrimitiveType.Bool, buffer); @@ -103,31 +121,30 @@ public class Lexer return new IdentifierToken(buffer); } - if (char.IsDigit(current.Value)) + if (char.IsDigit(current)) { var isFloat = false; var buffer = string.Empty; - - while (current.HasValue) + + while (Peek().TryGetValue(out var next)) { - if (current.Value == '.') + if (next == '.') { if (isFloat) { throw new Exception("More than one period found in float literal"); } + isFloat = true; - buffer += current.Value; + buffer += next; Next(); - current = Peek(); - } - else if (char.IsDigit(current.Value)) - { - buffer += current.Value; - Next(); - current = Peek(); } - else if (current.Value == 'f') + else if (char.IsDigit(next)) + { + buffer += next; + Next(); + } + else if (next == 'f') { isFloat = true; Next(); @@ -138,15 +155,15 @@ public class Lexer break; } } - + return new LiteralToken(isFloat ? NubPrimitiveType.F64 : NubPrimitiveType.I64, buffer); } // TODO: Revisit this foreach (var chain in Chians) { - if (current.Value != chain.Key[0]) continue; - + if (current != chain.Key[0]) continue; + for (var i = 1; i < chain.Key.Length; i++) { var c = Peek(i); @@ -164,36 +181,38 @@ public class Lexer } } - if (Chars.TryGetValue(current.Value, out var charSymbol)) + if (Chars.TryGetValue(current, out var charSymbol)) { Next(); return new SymbolToken(charSymbol); } - if (current.Value == '"') + if (current == '"') { Next(); var buffer = string.Empty; - + while (true) { - current = Peek(); + if (!Peek().TryGetValue(out var next)) + { + throw new Exception("Unclosed string literal"); + } + + if (next == '"') + { + Next(); + break; + } + + buffer += next; Next(); - if (!current.HasValue) throw new Exception("Unclosed string literal"); - if (current.Value == '"') break; - buffer += current.Value; } - + return new LiteralToken(NubPrimitiveType.String, buffer); } - if (char.IsWhiteSpace(current.Value)) - { - Next(); - return new SymbolToken(Symbol.Whitespace); - } - - throw new Exception($"Unknown character {current.Value}"); + throw new Exception($"Unknown character {current}"); } private Optional Peek(int offset = 0) @@ -202,7 +221,7 @@ public class Lexer { return _src[_index + offset]; } - + return Optional.Empty(); } diff --git a/src/compiler/Nub.Lang/Frontend/Lexing/SymbolToken.cs b/src/compiler/Nub.Lang/Frontend/Lexing/SymbolToken.cs index 882f751..5a2bc47 100644 --- a/src/compiler/Nub.Lang/Frontend/Lexing/SymbolToken.cs +++ b/src/compiler/Nub.Lang/Frontend/Lexing/SymbolToken.cs @@ -7,7 +7,6 @@ public class SymbolToken(Symbol symbol) : Token public enum Symbol { - Whitespace, Import, Func, Return, diff --git a/src/compiler/Nub.Lang/Frontend/Parsing/ExternFuncDefinitionNode.cs b/src/compiler/Nub.Lang/Frontend/Parsing/ExternFuncDefinitionNode.cs index f841795..fd93829 100644 --- a/src/compiler/Nub.Lang/Frontend/Parsing/ExternFuncDefinitionNode.cs +++ b/src/compiler/Nub.Lang/Frontend/Parsing/ExternFuncDefinitionNode.cs @@ -1,10 +1,11 @@ namespace Nub.Lang.Frontend.Parsing; -public class ExternFuncDefinitionNode(string name, List parameters, Optional returnType) : DefinitionNode +public class ExternFuncDefinitionNode(string name, List parameters, Optional returnType, Optional documentation) : DefinitionNode { public string Name { get; } = name; public List Parameters { get; } = parameters; public Optional ReturnType { get; } = returnType; + public Optional Documentation { get; set; } = documentation; public override string ToString() => $"{Name}({string.Join(", ", Parameters.Select(p => p.ToString()))}){(ReturnType.HasValue ? ": " + ReturnType.Value : "")}"; } \ No newline at end of file diff --git a/src/compiler/Nub.Lang/Frontend/Parsing/LocalFuncDefinitionNode.cs b/src/compiler/Nub.Lang/Frontend/Parsing/LocalFuncDefinitionNode.cs index 6734a02..b21c197 100644 --- a/src/compiler/Nub.Lang/Frontend/Parsing/LocalFuncDefinitionNode.cs +++ b/src/compiler/Nub.Lang/Frontend/Parsing/LocalFuncDefinitionNode.cs @@ -1,12 +1,13 @@ namespace Nub.Lang.Frontend.Parsing; -public class LocalFuncDefinitionNode(string name, List parameters, BlockNode body, Optional returnType, bool global) : DefinitionNode +public class LocalFuncDefinitionNode(string name, List parameters, BlockNode body, Optional returnType, bool global, Optional documentation) : DefinitionNode { public string Name { get; } = name; public List Parameters { get; } = parameters; public BlockNode Body { get; } = body; public Optional ReturnType { get; } = returnType; public bool Global { get; } = global; + public Optional Documentation { get; set; } = documentation; public override string ToString() => $"{Name}({string.Join(", ", Parameters.Select(p => p.ToString()))}){(ReturnType.HasValue ? ": " + ReturnType.Value : "")}"; } \ No newline at end of file diff --git a/src/compiler/Nub.Lang/Frontend/Parsing/Parser.cs b/src/compiler/Nub.Lang/Frontend/Parsing/Parser.cs index e4b1234..2dcd77f 100644 --- a/src/compiler/Nub.Lang/Frontend/Parsing/Parser.cs +++ b/src/compiler/Nub.Lang/Frontend/Parsing/Parser.cs @@ -36,6 +36,15 @@ public class Parser { List modifiers = []; + List documentationParts = []; + while (_index < _tokens.Count && _tokens[_index] is CommentToken commentToken) + { + documentationParts.Add(commentToken.Comment); + _index++; + } + + var documentation = documentationParts.Count == 0 ? null : string.Join('\n', documentationParts); + while (TryExpectModifier(out var modifier)) { modifiers.Add(modifier); @@ -44,13 +53,13 @@ public class Parser var keyword = ExpectSymbol(); return keyword.Symbol switch { - Symbol.Func => ParseFuncDefinition(modifiers), - Symbol.Struct => ParseStruct(modifiers), + Symbol.Func => ParseFuncDefinition(modifiers, Optional.OfNullable(documentation)), + Symbol.Struct => ParseStruct(modifiers, Optional.OfNullable(documentation)), _ => throw new Exception("Unexpected symbol: " + keyword.Symbol) }; } - private DefinitionNode ParseFuncDefinition(List modifiers) + private DefinitionNode ParseFuncDefinition(List modifiers, Optional documentation) { var name = ExpectIdentifier(); List parameters = []; @@ -77,7 +86,7 @@ public class Parser throw new Exception($"Modifiers: {string.Join(", ", modifiers)} is not valid for an extern function"); } - return new ExternFuncDefinitionNode(name.Value, parameters, returnType); + return new ExternFuncDefinitionNode(name.Value, parameters, returnType, documentation); } var body = ParseBlock(); @@ -88,10 +97,10 @@ public class Parser throw new Exception($"Modifiers: {string.Join(", ", modifiers)} is not valid for a local function"); } - return new LocalFuncDefinitionNode(name.Value, parameters, body, returnType, global); + return new LocalFuncDefinitionNode(name.Value, parameters, body, returnType, global, documentation); } - private StructDefinitionNode ParseStruct(List _) + private StructDefinitionNode ParseStruct(List _, Optional documentation) { var name = ExpectIdentifier().Value; @@ -115,7 +124,7 @@ public class Parser variables.Add(new StructField(variableName, variableType, variableValue)); } - return new StructDefinitionNode(name, variables); + return new StructDefinitionNode(name, variables, documentation); } private FuncParameter ParseFuncParameter() @@ -230,7 +239,7 @@ public class Parser private ExpressionNode ParseExpression(int precedence = 0) { var left = ParsePrimaryExpression(); - + while (true) { var token = Peek(); @@ -308,7 +317,7 @@ public class Parser private ExpressionNode ParsePrimaryExpression() { ExpressionNode expr; - + var token = ExpectToken(); switch (token) { @@ -341,6 +350,7 @@ public class Parser break; } } + break; } case SymbolToken symbolToken: @@ -403,6 +413,7 @@ public class Parser throw new Exception($"Unknown symbol: {symbolToken.Symbol}"); } } + break; } default: @@ -463,7 +474,7 @@ public class Parser { return NubType.Parse(name); } - + if (TryExpectSymbol(Symbol.Caret)) { var baseType = ParseType(); @@ -558,14 +569,15 @@ public class Parser private Optional Peek() { - while (_index < _tokens.Count && _tokens.ElementAt(_index) is SymbolToken { Symbol: Symbol.Whitespace }) + var peekIndex = _index; + while (peekIndex < _tokens.Count && _tokens[peekIndex] is CommentToken) { - Next(); + peekIndex++; } - - if (_index < _tokens.Count) + + if (peekIndex < _tokens.Count) { - return _tokens.ElementAt(_index); + return _tokens[peekIndex]; } return Optional.Empty(); @@ -573,6 +585,11 @@ public class Parser private void Next() { + while (_index < _tokens.Count && _tokens[_index] is CommentToken) + { + _index++; + } + _index++; } } \ No newline at end of file diff --git a/src/compiler/Nub.Lang/Frontend/Parsing/StructDefinitionNode.cs b/src/compiler/Nub.Lang/Frontend/Parsing/StructDefinitionNode.cs index dca6eab..43ea0a2 100644 --- a/src/compiler/Nub.Lang/Frontend/Parsing/StructDefinitionNode.cs +++ b/src/compiler/Nub.Lang/Frontend/Parsing/StructDefinitionNode.cs @@ -1,7 +1,8 @@ namespace Nub.Lang.Frontend.Parsing; -public class StructDefinitionNode(string name, List fields) : DefinitionNode +public class StructDefinitionNode(string name, List fields, Optional documentation) : DefinitionNode { public string Name { get; } = name; public List Fields { get; } = fields; + public Optional Documentation { get; set; } = documentation; } \ No newline at end of file diff --git a/src/compiler/Nub.Lang/Optional.cs b/src/compiler/Nub.Lang/Optional.cs index 439c87d..b60f923 100644 --- a/src/compiler/Nub.Lang/Optional.cs +++ b/src/compiler/Nub.Lang/Optional.cs @@ -44,5 +44,19 @@ public readonly struct Optional [MemberNotNullWhen(true, nameof(Value))] public bool HasValue { get; } + + [MemberNotNullWhen(true, nameof(Value))] + public bool TryGetValue([NotNullWhen(true)] out TValue? value) + { + if (HasValue) + { + value = Value; + return true; + } + + value = default; + return false; + } + public static implicit operator Optional(TValue value) => new(value); } \ No newline at end of file