...

src/compiler/Nub.Lang/Frontend/Lexing/Lexer.cs

@@ -3,7 +3,7 @@
 public class Lexer
 {
     private static readonly Dictionary<string, Symbol> Keywords = new()
-    { 
+    {
         ["func"] = Symbol.Func,
         ["import"] = Symbol.Import,
         ["if"] = Symbol.If,
@@ -15,9 +15,9 @@ public class Lexer
         ["new"] = Symbol.New,
         ["struct"] = Symbol.Struct,
     };
-    
+
     private static readonly Dictionary<string, Modifier> Modifers = new()
-    { 
+    {
         ["global"] = Modifier.Global,
         ["extern"] = Modifier.Extern,
     };
@@ -29,7 +29,7 @@ public class Lexer
         [['<', '=']] = Symbol.LessThanOrEqual,
         [['>', '=']] = Symbol.GreaterThanOrEqual,
     };
-    
+
     private static readonly Dictionary<char, Symbol> Chars = new()
     {
         [';'] = Symbol.Semicolon,
@@ -53,7 +53,7 @@ public class Lexer
         ['^'] = Symbol.Caret,
         ['&'] = Symbol.Ampersand,
     };
-    
+
     private string _src = string.Empty;
     private int _index;
 
@@ -61,40 +61,58 @@ public class Lexer
     {
         _src = src;
         _index = 0;
-        
+
         List<Token> tokens = [];
-        while (Peek().HasValue)
+        while (Peek().TryGetValue(out var character))
         {
-            tokens.Add(ParseToken());
+            if (char.IsWhiteSpace(character))
+            {
+                Next();
+                continue;
+            }
+            tokens.Add(ParseToken(character));
         }
+
         return tokens;
     }
 
-    private Token ParseToken()
+    private Token ParseToken(char current)
     {
-        var current = Peek();
-        
-        if (char.IsLetter(current.Value) || current.Value == '_')
+        if (current == '/' && Peek(1) is { Value: '/' })
+        {
+            Next();
+            Next();
+            var buffer = string.Empty;
+            while (Peek() is not { Value: '\n' })
+            {
+                buffer += Peek().Value;
+                Next();
+            }
+
+            Next();
+            return new CommentToken(buffer);
+        }
+
+        if (char.IsLetter(current) || current == '_')
         {
             var buffer = string.Empty;
-                
-            while (current.HasValue && (char.IsLetterOrDigit(current.Value) || current.Value == '_'))
+
+            while (Peek().TryGetValue(out var next) && (char.IsLetterOrDigit(next) || next == '_'))
             {
-                buffer += current.Value;
+                buffer += next;
                 Next();
-                current = Peek();
             }
 
             if (Keywords.TryGetValue(buffer, out var keywordSymbol))
             {
                 return new SymbolToken(keywordSymbol);
             }
-            
+
             if (Modifers.TryGetValue(buffer, out var modifer))
             {
                 return new ModifierToken(modifer);
             }
-            
+
             if (buffer is "true" or "false")
             {
                 return new LiteralToken(NubPrimitiveType.Bool, buffer);
@@ -103,31 +121,30 @@ public class Lexer
             return new IdentifierToken(buffer);
         }
 
-        if (char.IsDigit(current.Value))
+        if (char.IsDigit(current))
         {
             var isFloat = false;
             var buffer = string.Empty;
-                
-            while (current.HasValue)
+
+            while (Peek().TryGetValue(out var next))
             {
-                if (current.Value == '.')
+                if (next == '.')
                 {
                     if (isFloat)
                     {
                         throw new Exception("More than one period found in float literal");
                     }
+
                     isFloat = true;
-                    buffer += current.Value;
+                    buffer += next;
                     Next();
-                current = Peek();
-                } 
-                else if (char.IsDigit(current.Value))
-                {
-                    buffer += current.Value;
-                    Next();
-                    current = Peek();
                 }
-                else if (current.Value == 'f')
+                else if (char.IsDigit(next))
+                {
+                    buffer += next;
+                    Next();
+                }
+                else if (next == 'f')
                 {
                     isFloat = true;
                     Next();
@@ -138,15 +155,15 @@ public class Lexer
                     break;
                 }
             }
-            
+
             return new LiteralToken(isFloat ? NubPrimitiveType.F64 : NubPrimitiveType.I64, buffer);
         }
 
         // TODO: Revisit this
         foreach (var chain in Chians)
         {
-            if (current.Value != chain.Key[0]) continue;
-            
+            if (current != chain.Key[0]) continue;
+
             for (var i = 1; i < chain.Key.Length; i++)
             {
                 var c = Peek(i);
@@ -164,36 +181,38 @@ public class Lexer
             }
         }
 
-        if (Chars.TryGetValue(current.Value, out var charSymbol))
+        if (Chars.TryGetValue(current, out var charSymbol))
         {
             Next();
             return new SymbolToken(charSymbol);
         }
 
-        if (current.Value == '"')
+        if (current == '"')
         {
             Next();
             var buffer = string.Empty;
-            
+
             while (true)
             {
-                current = Peek();
+                if (!Peek().TryGetValue(out var next))
+                {
+                    throw new Exception("Unclosed string literal");
+                }
+                
+                if (next == '"')
+                {
+                    Next();
+                    break;
+                }
+                
+                buffer += next;
                 Next();
-                if (!current.HasValue) throw new Exception("Unclosed string literal");
-                if (current.Value == '"') break;
-                buffer += current.Value;
             }
-            
+
             return new LiteralToken(NubPrimitiveType.String, buffer);
         }
 
-        if (char.IsWhiteSpace(current.Value))
-        {
-            Next();
-            return new SymbolToken(Symbol.Whitespace);
-        }
-
-        throw new Exception($"Unknown character {current.Value}");
+        throw new Exception($"Unknown character {current}");
     }
 
     private Optional<char> Peek(int offset = 0)
@@ -202,7 +221,7 @@ public class Lexer
         {
             return _src[_index + offset];
         }
-        
+
         return Optional<char>.Empty();
     }