...

example/program.nub

@@ -1,9 +1,14 @@
 import c
 
+// Test
+// Test2
+// Test3
+// Test4
 global func main(args: []string) {
     i = 0
     printf("%d\n", args.count)
     while i < args.count {
+        // Test
         printf("%s\n", args[i])
         i = i + 1
     }

src/compiler/Nub.Lang/Frontend/Lexing/CommentToken.cs

@@ -0,0 +1,6 @@
+namespace Nub.Lang.Frontend.Lexing;
+
+public class CommentToken(string comment) : Token
+{
+    public string Comment { get; } = comment;
+}

src/compiler/Nub.Lang/Frontend/Lexing/Lexer.cs

@@ -3,7 +3,7 @@
 public class Lexer
 {
     private static readonly Dictionary<string, Symbol> Keywords = new()
-    { 
+    {
         ["func"] = Symbol.Func,
         ["import"] = Symbol.Import,
         ["if"] = Symbol.If,
@@ -15,9 +15,9 @@ public class Lexer
         ["new"] = Symbol.New,
         ["struct"] = Symbol.Struct,
     };
-    
+
     private static readonly Dictionary<string, Modifier> Modifers = new()
-    { 
+    {
         ["global"] = Modifier.Global,
         ["extern"] = Modifier.Extern,
     };
@@ -29,7 +29,7 @@ public class Lexer
         [['<', '=']] = Symbol.LessThanOrEqual,
         [['>', '=']] = Symbol.GreaterThanOrEqual,
     };
-    
+
     private static readonly Dictionary<char, Symbol> Chars = new()
     {
         [';'] = Symbol.Semicolon,
@@ -53,7 +53,7 @@ public class Lexer
         ['^'] = Symbol.Caret,
         ['&'] = Symbol.Ampersand,
     };
-    
+
     private string _src = string.Empty;
     private int _index;
 
@@ -61,40 +61,58 @@ public class Lexer
     {
         _src = src;
         _index = 0;
-        
+
         List<Token> tokens = [];
-        while (Peek().HasValue)
+        while (Peek().TryGetValue(out var character))
         {
-            tokens.Add(ParseToken());
+            if (char.IsWhiteSpace(character))
+            {
+                Next();
+                continue;
+            }
+            tokens.Add(ParseToken(character));
         }
+
         return tokens;
     }
 
-    private Token ParseToken()
+    private Token ParseToken(char current)
     {
-        var current = Peek();
+        if (current == '/' && Peek(1) is { Value: '/' })
-        
+        {
-        if (char.IsLetter(current.Value) || current.Value == '_')
+            Next();
+            Next();
+            var buffer = string.Empty;
+            while (Peek() is not { Value: '\n' })
+            {
+                buffer += Peek().Value;
+                Next();
+            }
+
+            Next();
+            return new CommentToken(buffer);
+        }
+
+        if (char.IsLetter(current) || current == '_')
         {
             var buffer = string.Empty;
-                
+
-            while (current.HasValue && (char.IsLetterOrDigit(current.Value) || current.Value == '_'))
+            while (Peek().TryGetValue(out var next) && (char.IsLetterOrDigit(next) || next == '_'))
             {
-                buffer += current.Value;
+                buffer += next;
                 Next();
-                current = Peek();
             }
 
             if (Keywords.TryGetValue(buffer, out var keywordSymbol))
             {
                 return new SymbolToken(keywordSymbol);
             }
-            
+
             if (Modifers.TryGetValue(buffer, out var modifer))
             {
                 return new ModifierToken(modifer);
             }
-            
+
             if (buffer is "true" or "false")
             {
                 return new LiteralToken(NubPrimitiveType.Bool, buffer);
@@ -103,31 +121,30 @@ public class Lexer
             return new IdentifierToken(buffer);
         }
 
-        if (char.IsDigit(current.Value))
+        if (char.IsDigit(current))
         {
             var isFloat = false;
             var buffer = string.Empty;
-                
+
-            while (current.HasValue)
+            while (Peek().TryGetValue(out var next))
             {
-                if (current.Value == '.')
+                if (next == '.')
                 {
                     if (isFloat)
                     {
                         throw new Exception("More than one period found in float literal");
                     }
+
                     isFloat = true;
-                    buffer += current.Value;
+                    buffer += next;
                     Next();
-                current = Peek();
-                } 
-                else if (char.IsDigit(current.Value))
-                {
-                    buffer += current.Value;
-                    Next();
-                    current = Peek();
                 }
-                else if (current.Value == 'f')
+                else if (char.IsDigit(next))
+                {
+                    buffer += next;
+                    Next();
+                }
+                else if (next == 'f')
                 {
                     isFloat = true;
                     Next();
@@ -138,15 +155,15 @@ public class Lexer
                     break;
                 }
             }
-            
+
             return new LiteralToken(isFloat ? NubPrimitiveType.F64 : NubPrimitiveType.I64, buffer);
         }
 
         // TODO: Revisit this
         foreach (var chain in Chians)
         {
-            if (current.Value != chain.Key[0]) continue;
+            if (current != chain.Key[0]) continue;
-            
+
             for (var i = 1; i < chain.Key.Length; i++)
             {
                 var c = Peek(i);
@@ -164,36 +181,38 @@ public class Lexer
             }
         }
 
-        if (Chars.TryGetValue(current.Value, out var charSymbol))
+        if (Chars.TryGetValue(current, out var charSymbol))
         {
             Next();
             return new SymbolToken(charSymbol);
         }
 
-        if (current.Value == '"')
+        if (current == '"')
         {
             Next();
             var buffer = string.Empty;
-            
+
             while (true)
             {
-                current = Peek();
+                if (!Peek().TryGetValue(out var next))
+                {
+                    throw new Exception("Unclosed string literal");
+                }
+                
+                if (next == '"')
+                {
+                    Next();
+                    break;
+                }
+                
+                buffer += next;
                 Next();
-                if (!current.HasValue) throw new Exception("Unclosed string literal");
-                if (current.Value == '"') break;
-                buffer += current.Value;
             }
-            
+
             return new LiteralToken(NubPrimitiveType.String, buffer);
         }
 
-        if (char.IsWhiteSpace(current.Value))
+        throw new Exception($"Unknown character {current}");
-        {
-            Next();
-            return new SymbolToken(Symbol.Whitespace);
-        }
-
-        throw new Exception($"Unknown character {current.Value}");
     }
 
     private Optional<char> Peek(int offset = 0)
@@ -202,7 +221,7 @@ public class Lexer
         {
             return _src[_index + offset];
         }
-        
+
         return Optional<char>.Empty();
     }
 

src/compiler/Nub.Lang/Frontend/Lexing/SymbolToken.cs

@@ -7,7 +7,6 @@ public class SymbolToken(Symbol symbol) : Token
 
 public enum Symbol
 {
-    Whitespace,
     Import,
     Func,
     Return,

src/compiler/Nub.Lang/Frontend/Parsing/ExternFuncDefinitionNode.cs

@@ -1,10 +1,11 @@
 namespace Nub.Lang.Frontend.Parsing;
 
-public class ExternFuncDefinitionNode(string name, List<FuncParameter> parameters, Optional<NubType> returnType) : DefinitionNode
+public class ExternFuncDefinitionNode(string name, List<FuncParameter> parameters, Optional<NubType> returnType, Optional<string> documentation) : DefinitionNode
 {
     public string Name { get; } = name;
     public List<FuncParameter> Parameters { get; } = parameters;
     public Optional<NubType> ReturnType { get; } = returnType;
+    public Optional<string> Documentation { get; set; } = documentation;
 
     public override string ToString() => $"{Name}({string.Join(", ", Parameters.Select(p => p.ToString()))}){(ReturnType.HasValue ? ": " + ReturnType.Value : "")}";
 }

src/compiler/Nub.Lang/Frontend/Parsing/LocalFuncDefinitionNode.cs

@@ -1,12 +1,13 @@
 namespace Nub.Lang.Frontend.Parsing;
 
-public class LocalFuncDefinitionNode(string name, List<FuncParameter> parameters, BlockNode body, Optional<NubType> returnType, bool global) : DefinitionNode
+public class LocalFuncDefinitionNode(string name, List<FuncParameter> parameters, BlockNode body, Optional<NubType> returnType, bool global, Optional<string> documentation) : DefinitionNode
 {
     public string Name { get; } = name;
     public List<FuncParameter> Parameters { get; } = parameters;
     public BlockNode Body { get; } = body;
     public Optional<NubType> ReturnType { get; } = returnType;
     public bool Global { get; } = global;
+    public Optional<string> Documentation { get; set; } = documentation;
 
     public override string ToString() => $"{Name}({string.Join(", ", Parameters.Select(p => p.ToString()))}){(ReturnType.HasValue ? ": " + ReturnType.Value : "")}";
 }

src/compiler/Nub.Lang/Frontend/Parsing/Parser.cs

@@ -36,6 +36,15 @@ public class Parser
     {
         List<Modifier> modifiers = [];
 
+        List<string> documentationParts = [];
+        while (_index < _tokens.Count && _tokens[_index] is CommentToken commentToken)
+        {
+            documentationParts.Add(commentToken.Comment);
+            _index++;
+        }
+        
+        var documentation = documentationParts.Count == 0 ? null : string.Join('\n', documentationParts);
+        
         while (TryExpectModifier(out var modifier))
         {
             modifiers.Add(modifier);
@@ -44,13 +53,13 @@ public class Parser
         var keyword = ExpectSymbol();
         return keyword.Symbol switch
         {
-            Symbol.Func => ParseFuncDefinition(modifiers),
+            Symbol.Func => ParseFuncDefinition(modifiers, Optional.OfNullable(documentation)),
-            Symbol.Struct => ParseStruct(modifiers),
+            Symbol.Struct => ParseStruct(modifiers, Optional.OfNullable(documentation)),
             _ => throw new Exception("Unexpected symbol: " + keyword.Symbol)
         };
     }
 
-    private DefinitionNode ParseFuncDefinition(List<Modifier> modifiers)
+    private DefinitionNode ParseFuncDefinition(List<Modifier> modifiers, Optional<string> documentation)
     {
         var name = ExpectIdentifier();
         List<FuncParameter> parameters = [];
@@ -77,7 +86,7 @@ public class Parser
                 throw new Exception($"Modifiers: {string.Join(", ", modifiers)} is not valid for an extern function");
             }
 
-            return new ExternFuncDefinitionNode(name.Value, parameters, returnType);
+            return new ExternFuncDefinitionNode(name.Value, parameters, returnType, documentation);
         }
 
         var body = ParseBlock();
@@ -88,10 +97,10 @@ public class Parser
             throw new Exception($"Modifiers: {string.Join(", ", modifiers)} is not valid for a local function");
         }
 
-        return new LocalFuncDefinitionNode(name.Value, parameters, body, returnType, global);
+        return new LocalFuncDefinitionNode(name.Value, parameters, body, returnType, global, documentation);
     }
 
-    private StructDefinitionNode ParseStruct(List<Modifier> _)
+    private StructDefinitionNode ParseStruct(List<Modifier> _, Optional<string> documentation)
     {
         var name = ExpectIdentifier().Value;
 
@@ -115,7 +124,7 @@ public class Parser
             variables.Add(new StructField(variableName, variableType, variableValue));
         }
 
-        return new StructDefinitionNode(name, variables);
+        return new StructDefinitionNode(name, variables, documentation);
     }
 
     private FuncParameter ParseFuncParameter()
@@ -230,7 +239,7 @@ public class Parser
     private ExpressionNode ParseExpression(int precedence = 0)
     {
         var left = ParsePrimaryExpression();
-        
+
         while (true)
         {
             var token = Peek();
@@ -308,7 +317,7 @@ public class Parser
     private ExpressionNode ParsePrimaryExpression()
     {
         ExpressionNode expr;
-        
+
         var token = ExpectToken();
         switch (token)
         {
@@ -341,6 +350,7 @@ public class Parser
                         break;
                     }
                 }
+
                 break;
             }
             case SymbolToken symbolToken:
@@ -403,6 +413,7 @@ public class Parser
                         throw new Exception($"Unknown symbol: {symbolToken.Symbol}");
                     }
                 }
+
                 break;
             }
             default:
@@ -463,7 +474,7 @@ public class Parser
         {
             return NubType.Parse(name);
         }
-        
+
         if (TryExpectSymbol(Symbol.Caret))
         {
             var baseType = ParseType();
@@ -558,14 +569,15 @@ public class Parser
 
     private Optional<Token> Peek()
     {
-        while (_index < _tokens.Count && _tokens.ElementAt(_index) is SymbolToken { Symbol: Symbol.Whitespace })
+        var peekIndex = _index;
+        while (peekIndex < _tokens.Count && _tokens[peekIndex] is CommentToken)
         {
-            Next();
+            peekIndex++;
         }
-
+        
-        if (_index < _tokens.Count)
+        if (peekIndex < _tokens.Count)
         {
-            return _tokens.ElementAt(_index);
+            return _tokens[peekIndex];
         }
 
         return Optional<Token>.Empty();
@@ -573,6 +585,11 @@ public class Parser
 
     private void Next()
     {
+        while (_index < _tokens.Count && _tokens[_index] is CommentToken)
+        {
+            _index++;
+        }
+        
         _index++;
     }
 }

src/compiler/Nub.Lang/Frontend/Parsing/StructDefinitionNode.cs

@@ -1,7 +1,8 @@
 namespace Nub.Lang.Frontend.Parsing;
 
-public class StructDefinitionNode(string name, List<StructField> fields) : DefinitionNode
+public class StructDefinitionNode(string name, List<StructField> fields, Optional<string> documentation) : DefinitionNode
 {
     public string Name { get; } = name;
     public List<StructField> Fields { get; } = fields;
+    public Optional<string> Documentation { get; set; } = documentation;
 }

src/compiler/Nub.Lang/Optional.cs

@@ -44,5 +44,19 @@ public readonly struct Optional<TValue>
     [MemberNotNullWhen(true, nameof(Value))]
     public bool HasValue { get; }
 
+
+    [MemberNotNullWhen(true, nameof(Value))]
+    public bool TryGetValue([NotNullWhen(true)] out TValue? value)
+    {
+        if (HasValue)
+        {
+            value = Value;
+            return true;
+        }
+
+        value = default;
+        return false;
+    }
+
     public static implicit operator Optional<TValue>(TValue value) => new(value);
 }