From 0cb10a68f66750f342b5d04a441177d2449c71dd Mon Sep 17 00:00:00 2001 From: nub31 Date: Sat, 24 May 2025 20:59:26 +0200 Subject: [PATCH] docs parsing --- .../Nub.Lang/Frontend/Lexing/CommentToken.cs | 6 -- .../Frontend/Lexing/DocumentationToken.cs | 6 ++ .../Nub.Lang/Frontend/Lexing/Lexer.cs | 82 ++++++++++++++----- .../Frontend/Parsing/DefinitionNode.cs | 5 +- .../Parsing/ExternFuncDefinitionNode.cs | 3 +- .../Parsing/LocalFuncDefinitionNode.cs | 3 +- .../Nub.Lang/Frontend/Parsing/Parser.cs | 16 ++-- .../Frontend/Parsing/StructDefinitionNode.cs | 3 +- 8 files changed, 84 insertions(+), 40 deletions(-) delete mode 100644 src/compiler/Nub.Lang/Frontend/Lexing/CommentToken.cs create mode 100644 src/compiler/Nub.Lang/Frontend/Lexing/DocumentationToken.cs diff --git a/src/compiler/Nub.Lang/Frontend/Lexing/CommentToken.cs b/src/compiler/Nub.Lang/Frontend/Lexing/CommentToken.cs deleted file mode 100644 index f4568f1..0000000 --- a/src/compiler/Nub.Lang/Frontend/Lexing/CommentToken.cs +++ /dev/null @@ -1,6 +0,0 @@ -namespace Nub.Lang.Frontend.Lexing; - -public class CommentToken(string comment) : Token -{ - public string Comment { get; } = comment; -} \ No newline at end of file diff --git a/src/compiler/Nub.Lang/Frontend/Lexing/DocumentationToken.cs b/src/compiler/Nub.Lang/Frontend/Lexing/DocumentationToken.cs new file mode 100644 index 0000000..7c28db2 --- /dev/null +++ b/src/compiler/Nub.Lang/Frontend/Lexing/DocumentationToken.cs @@ -0,0 +1,6 @@ +namespace Nub.Lang.Frontend.Lexing; + +public class DocumentationToken(string documentation) : Token +{ + public string Documentation { get; } = documentation; +} \ No newline at end of file diff --git a/src/compiler/Nub.Lang/Frontend/Lexing/Lexer.cs b/src/compiler/Nub.Lang/Frontend/Lexing/Lexer.cs index 0a41049..125727d 100644 --- a/src/compiler/Nub.Lang/Frontend/Lexing/Lexer.cs +++ b/src/compiler/Nub.Lang/Frontend/Lexing/Lexer.cs @@ -63,34 +63,78 @@ public class Lexer _index = 0; List tokens = []; - while (Peek().TryGetValue(out var character)) + while (ParseToken().TryGetValue(out var token)) { - if (char.IsWhiteSpace(character)) - { - Next(); - continue; - } - tokens.Add(ParseToken(character)); + tokens.Add(token); } return tokens; } - private Token ParseToken(char current) + private void ConsumeWhitespace() { - if (current == '/' && Peek(1) is { Value: '/' }) + while (Peek().TryGetValue(out var character) && char.IsWhiteSpace(character)) + { + Next(); + } + } + + private Optional ParseToken() + { + ConsumeWhitespace(); + + string? documentation = null; + while (Peek() is { Value: '/' } && Peek(1) is { Value: '/' }) { Next(); Next(); - var buffer = string.Empty; - while (Peek() is not { Value: '\n' }) - { - buffer += Peek().Value; - Next(); - } - Next(); - return new CommentToken(buffer); + if (Peek() is { Value: '/' }) + { + Next(); + + if (documentation != null) + { + documentation += '\n'; + } + + while (Peek().TryGetValue(out var character)) + { + if (character == '\n') + { + Next(); + break; + } + + documentation += character; + Next(); + } + } + else + { + while (Peek().TryGetValue(out var character)) + { + if (character == '\n') + { + Next(); + break; + } + + Next(); + } + } + } + + if (documentation != null) + { + return new DocumentationToken(documentation); + } + + ConsumeWhitespace(); + + if (!Peek().TryGetValue(out var current)) + { + return Optional.Empty(); } if (char.IsLetter(current) || current == '_') @@ -198,13 +242,13 @@ public class Lexer { throw new Exception("Unclosed string literal"); } - + if (next == '"') { Next(); break; } - + buffer += next; Next(); } diff --git a/src/compiler/Nub.Lang/Frontend/Parsing/DefinitionNode.cs b/src/compiler/Nub.Lang/Frontend/Parsing/DefinitionNode.cs index b0d239a..6077565 100644 --- a/src/compiler/Nub.Lang/Frontend/Parsing/DefinitionNode.cs +++ b/src/compiler/Nub.Lang/Frontend/Parsing/DefinitionNode.cs @@ -1,3 +1,6 @@ namespace Nub.Lang.Frontend.Parsing; -public abstract class DefinitionNode : Node; \ No newline at end of file +public abstract class DefinitionNode(Optional documentation) : Node +{ + public Optional Documentation { get; set; } = documentation; +} \ No newline at end of file diff --git a/src/compiler/Nub.Lang/Frontend/Parsing/ExternFuncDefinitionNode.cs b/src/compiler/Nub.Lang/Frontend/Parsing/ExternFuncDefinitionNode.cs index fd93829..b3ab11a 100644 --- a/src/compiler/Nub.Lang/Frontend/Parsing/ExternFuncDefinitionNode.cs +++ b/src/compiler/Nub.Lang/Frontend/Parsing/ExternFuncDefinitionNode.cs @@ -1,11 +1,10 @@ namespace Nub.Lang.Frontend.Parsing; -public class ExternFuncDefinitionNode(string name, List parameters, Optional returnType, Optional documentation) : DefinitionNode +public class ExternFuncDefinitionNode(string name, List parameters, Optional returnType, Optional documentation) : DefinitionNode(documentation) { public string Name { get; } = name; public List Parameters { get; } = parameters; public Optional ReturnType { get; } = returnType; - public Optional Documentation { get; set; } = documentation; public override string ToString() => $"{Name}({string.Join(", ", Parameters.Select(p => p.ToString()))}){(ReturnType.HasValue ? ": " + ReturnType.Value : "")}"; } \ No newline at end of file diff --git a/src/compiler/Nub.Lang/Frontend/Parsing/LocalFuncDefinitionNode.cs b/src/compiler/Nub.Lang/Frontend/Parsing/LocalFuncDefinitionNode.cs index b21c197..8e987c4 100644 --- a/src/compiler/Nub.Lang/Frontend/Parsing/LocalFuncDefinitionNode.cs +++ b/src/compiler/Nub.Lang/Frontend/Parsing/LocalFuncDefinitionNode.cs @@ -1,13 +1,12 @@ namespace Nub.Lang.Frontend.Parsing; -public class LocalFuncDefinitionNode(string name, List parameters, BlockNode body, Optional returnType, bool global, Optional documentation) : DefinitionNode +public class LocalFuncDefinitionNode(string name, List parameters, BlockNode body, Optional returnType, bool global, Optional documentation) : DefinitionNode(documentation) { public string Name { get; } = name; public List Parameters { get; } = parameters; public BlockNode Body { get; } = body; public Optional ReturnType { get; } = returnType; public bool Global { get; } = global; - public Optional Documentation { get; set; } = documentation; public override string ToString() => $"{Name}({string.Join(", ", Parameters.Select(p => p.ToString()))}){(ReturnType.HasValue ? ": " + ReturnType.Value : "")}"; } \ No newline at end of file diff --git a/src/compiler/Nub.Lang/Frontend/Parsing/Parser.cs b/src/compiler/Nub.Lang/Frontend/Parsing/Parser.cs index 2dcd77f..af55174 100644 --- a/src/compiler/Nub.Lang/Frontend/Parsing/Parser.cs +++ b/src/compiler/Nub.Lang/Frontend/Parsing/Parser.cs @@ -37,14 +37,14 @@ public class Parser List modifiers = []; List documentationParts = []; - while (_index < _tokens.Count && _tokens[_index] is CommentToken commentToken) + while (_index < _tokens.Count && _tokens[_index] is DocumentationToken commentToken) { - documentationParts.Add(commentToken.Comment); + documentationParts.Add(commentToken.Documentation); _index++; } - + var documentation = documentationParts.Count == 0 ? null : string.Join('\n', documentationParts); - + while (TryExpectModifier(out var modifier)) { modifiers.Add(modifier); @@ -570,11 +570,11 @@ public class Parser private Optional Peek() { var peekIndex = _index; - while (peekIndex < _tokens.Count && _tokens[peekIndex] is CommentToken) + while (peekIndex < _tokens.Count && _tokens[peekIndex] is DocumentationToken) { peekIndex++; } - + if (peekIndex < _tokens.Count) { return _tokens[peekIndex]; @@ -585,11 +585,11 @@ public class Parser private void Next() { - while (_index < _tokens.Count && _tokens[_index] is CommentToken) + while (_index < _tokens.Count && _tokens[_index] is DocumentationToken) { _index++; } - + _index++; } } \ No newline at end of file diff --git a/src/compiler/Nub.Lang/Frontend/Parsing/StructDefinitionNode.cs b/src/compiler/Nub.Lang/Frontend/Parsing/StructDefinitionNode.cs index 43ea0a2..43f3fa4 100644 --- a/src/compiler/Nub.Lang/Frontend/Parsing/StructDefinitionNode.cs +++ b/src/compiler/Nub.Lang/Frontend/Parsing/StructDefinitionNode.cs @@ -1,8 +1,7 @@ namespace Nub.Lang.Frontend.Parsing; -public class StructDefinitionNode(string name, List fields, Optional documentation) : DefinitionNode +public class StructDefinitionNode(string name, List fields, Optional documentation) : DefinitionNode(documentation) { public string Name { get; } = name; public List Fields { get; } = fields; - public Optional Documentation { get; set; } = documentation; } \ No newline at end of file