docs parsing

src/compiler/Nub.Lang/Frontend/Lexing/Lexer.cs

@@ -63,34 +63,78 @@ public class Lexer
         _index = 0;
 
         List<Token> tokens = [];
-        while (Peek().TryGetValue(out var character))
+        while (ParseToken().TryGetValue(out var token))
         {
-            if (char.IsWhiteSpace(character))
-            {
-                Next();
-                continue;
-            }
-            tokens.Add(ParseToken(character));
+            tokens.Add(token);
         }
 
         return tokens;
     }
 
-    private Token ParseToken(char current)
+    private void ConsumeWhitespace()
     {
-        if (current == '/' && Peek(1) is { Value: '/' })
+        while (Peek().TryGetValue(out var character) && char.IsWhiteSpace(character))
+        {
+            Next();
+        }
+    }
+
+    private Optional<Token> ParseToken()
+    {
+        ConsumeWhitespace();
+
+        string? documentation = null;
+        while (Peek() is { Value: '/' } && Peek(1) is { Value: '/' })
         {
             Next();
             Next();
-            var buffer = string.Empty;
-            while (Peek() is not { Value: '\n' })
-            {
-                buffer += Peek().Value;
-                Next();
-            }
 
-            Next();
-            return new CommentToken(buffer);
+            if (Peek() is { Value: '/' })
+            {
+                Next();
+
+                if (documentation != null)
+                {
+                    documentation += '\n';
+                }
+
+                while (Peek().TryGetValue(out var character))
+                {
+                    if (character == '\n')
+                    {
+                        Next();
+                        break;
+                    }
+
+                    documentation += character;
+                    Next();
+                }
+            }
+            else
+            {
+                while (Peek().TryGetValue(out var character))
+                {
+                    if (character == '\n')
+                    {
+                        Next();
+                        break;
+                    }
+
+                    Next();
+                }
+            }
+        }
+
+        if (documentation != null)
+        {
+            return new DocumentationToken(documentation);
+        }
+
+        ConsumeWhitespace();
+
+        if (!Peek().TryGetValue(out var current))
+        {
+            return Optional<Token>.Empty();
         }
 
         if (char.IsLetter(current) || current == '_')
@@ -198,13 +242,13 @@ public class Lexer
                 {
                     throw new Exception("Unclosed string literal");
                 }
-                
+
                 if (next == '"')
                 {
                     Next();
                     break;
                 }
-                
+
                 buffer += next;
                 Next();
             }