docs parsing

src/compiler/Nub.Lang/Frontend/Lexing/CommentToken.cs

@@ -1,6 +0,0 @@
-namespace Nub.Lang.Frontend.Lexing;
-
-public class CommentToken(string comment) : Token
-{
-    public string Comment { get; } = comment;
-}

src/compiler/Nub.Lang/Frontend/Lexing/DocumentationToken.cs

@@ -0,0 +1,6 @@
+namespace Nub.Lang.Frontend.Lexing;
+
+public class DocumentationToken(string documentation) : Token
+{
+    public string Documentation { get; } = documentation;
+}

src/compiler/Nub.Lang/Frontend/Lexing/Lexer.cs

@@ -63,34 +63,78 @@ public class Lexer
         _index = 0;
 
         List<Token> tokens = [];
-        while (Peek().TryGetValue(out var character))
+        while (ParseToken().TryGetValue(out var token))
         {
-            if (char.IsWhiteSpace(character))
+            tokens.Add(token);
-            {
-                Next();
-                continue;
-            }
-            tokens.Add(ParseToken(character));
         }
 
         return tokens;
     }
 
-    private Token ParseToken(char current)
+    private void ConsumeWhitespace()
     {
-        if (current == '/' && Peek(1) is { Value: '/' })
+        while (Peek().TryGetValue(out var character) && char.IsWhiteSpace(character))
+        {
+            Next();
+        }
+    }
+
+    private Optional<Token> ParseToken()
+    {
+        ConsumeWhitespace();
+
+        string? documentation = null;
+        while (Peek() is { Value: '/' } && Peek(1) is { Value: '/' })
         {
             Next();
             Next();
-            var buffer = string.Empty;
-            while (Peek() is not { Value: '\n' })
-            {
-                buffer += Peek().Value;
-                Next();
-            }
 
-            Next();
+            if (Peek() is { Value: '/' })
-            return new CommentToken(buffer);
+            {
+                Next();
+
+                if (documentation != null)
+                {
+                    documentation += '\n';
+                }
+
+                while (Peek().TryGetValue(out var character))
+                {
+                    if (character == '\n')
+                    {
+                        Next();
+                        break;
+                    }
+
+                    documentation += character;
+                    Next();
+                }
+            }
+            else
+            {
+                while (Peek().TryGetValue(out var character))
+                {
+                    if (character == '\n')
+                    {
+                        Next();
+                        break;
+                    }
+
+                    Next();
+                }
+            }
+        }
+
+        if (documentation != null)
+        {
+            return new DocumentationToken(documentation);
+        }
+
+        ConsumeWhitespace();
+
+        if (!Peek().TryGetValue(out var current))
+        {
+            return Optional<Token>.Empty();
         }
 
         if (char.IsLetter(current) || current == '_')
@@ -198,13 +242,13 @@ public class Lexer
                 {
                     throw new Exception("Unclosed string literal");
                 }
-                
+
                 if (next == '"')
                 {
                     Next();
                     break;
                 }
-                
+
                 buffer += next;
                 Next();
             }

src/compiler/Nub.Lang/Frontend/Parsing/DefinitionNode.cs

@@ -1,3 +1,6 @@
 namespace Nub.Lang.Frontend.Parsing;
 
-public abstract class DefinitionNode : Node;
+public abstract class DefinitionNode(Optional<string> documentation) : Node
+{
+    public Optional<string> Documentation { get; set; } = documentation;
+}

src/compiler/Nub.Lang/Frontend/Parsing/ExternFuncDefinitionNode.cs

@@ -1,11 +1,10 @@
 namespace Nub.Lang.Frontend.Parsing;
 
-public class ExternFuncDefinitionNode(string name, List<FuncParameter> parameters, Optional<NubType> returnType, Optional<string> documentation) : DefinitionNode
+public class ExternFuncDefinitionNode(string name, List<FuncParameter> parameters, Optional<NubType> returnType, Optional<string> documentation) : DefinitionNode(documentation)
 {
     public string Name { get; } = name;
     public List<FuncParameter> Parameters { get; } = parameters;
     public Optional<NubType> ReturnType { get; } = returnType;
-    public Optional<string> Documentation { get; set; } = documentation;
 
     public override string ToString() => $"{Name}({string.Join(", ", Parameters.Select(p => p.ToString()))}){(ReturnType.HasValue ? ": " + ReturnType.Value : "")}";
 }

src/compiler/Nub.Lang/Frontend/Parsing/LocalFuncDefinitionNode.cs

@@ -1,13 +1,12 @@
 namespace Nub.Lang.Frontend.Parsing;
 
-public class LocalFuncDefinitionNode(string name, List<FuncParameter> parameters, BlockNode body, Optional<NubType> returnType, bool global, Optional<string> documentation) : DefinitionNode
+public class LocalFuncDefinitionNode(string name, List<FuncParameter> parameters, BlockNode body, Optional<NubType> returnType, bool global, Optional<string> documentation) : DefinitionNode(documentation)
 {
     public string Name { get; } = name;
     public List<FuncParameter> Parameters { get; } = parameters;
     public BlockNode Body { get; } = body;
     public Optional<NubType> ReturnType { get; } = returnType;
     public bool Global { get; } = global;
-    public Optional<string> Documentation { get; set; } = documentation;
 
     public override string ToString() => $"{Name}({string.Join(", ", Parameters.Select(p => p.ToString()))}){(ReturnType.HasValue ? ": " + ReturnType.Value : "")}";
 }

src/compiler/Nub.Lang/Frontend/Parsing/Parser.cs

@@ -37,14 +37,14 @@ public class Parser
         List<Modifier> modifiers = [];
 
         List<string> documentationParts = [];
-        while (_index < _tokens.Count && _tokens[_index] is CommentToken commentToken)
+        while (_index < _tokens.Count && _tokens[_index] is DocumentationToken commentToken)
         {
-            documentationParts.Add(commentToken.Comment);
+            documentationParts.Add(commentToken.Documentation);
             _index++;
         }
-        
+
         var documentation = documentationParts.Count == 0 ? null : string.Join('\n', documentationParts);
-        
+
         while (TryExpectModifier(out var modifier))
         {
             modifiers.Add(modifier);
@@ -570,11 +570,11 @@ public class Parser
     private Optional<Token> Peek()
     {
         var peekIndex = _index;
-        while (peekIndex < _tokens.Count && _tokens[peekIndex] is CommentToken)
+        while (peekIndex < _tokens.Count && _tokens[peekIndex] is DocumentationToken)
         {
             peekIndex++;
         }
-        
+
         if (peekIndex < _tokens.Count)
         {
             return _tokens[peekIndex];
@@ -585,11 +585,11 @@ public class Parser
 
     private void Next()
     {
-        while (_index < _tokens.Count && _tokens[_index] is CommentToken)
+        while (_index < _tokens.Count && _tokens[_index] is DocumentationToken)
         {
             _index++;
         }
-        
+
         _index++;
     }
 }

src/compiler/Nub.Lang/Frontend/Parsing/StructDefinitionNode.cs

@@ -1,8 +1,7 @@
 namespace Nub.Lang.Frontend.Parsing;
 
-public class StructDefinitionNode(string name, List<StructField> fields, Optional<string> documentation) : DefinitionNode
+public class StructDefinitionNode(string name, List<StructField> fields, Optional<string> documentation) : DefinitionNode(documentation)
 {
     public string Name { get; } = name;
     public List<StructField> Fields { get; } = fields;
-    public Optional<string> Documentation { get; set; } = documentation;
 }