...

compiler/NubLang/Syntax/Tokenizer.cs

@@ -29,32 +29,7 @@ public sealed class Tokenizer
         {
             try
             {
-                var current = _content[_index];
-
-                if (char.IsWhiteSpace(current))
-                {
-                    if (current == '\n')
-                    {
-                        _line += 1;
-                        _column = 0;
-                    }
-
-                    Next();
-                    continue;
-                }
-
-                if (current == '/' && _index + 1 < _content.Length && _content[_index + 1] == '/')
-                {
-                    Next(2);
-                    while (_index < _content.Length && _content[_index] != '\n')
-                    {
-                        Next();
-                    }
-
-                    continue;
-                }
-
-                tokens.Add(ParseToken(current, _line, _column));
+                tokens.Add(ParseToken());
             }
             catch (CompileException e)
             {
@@ -66,14 +41,40 @@ public sealed class Tokenizer
         return tokens;
     }
 
-    private Token ParseToken(char current, int lineStart, int columnStart)
+    private Token ParseToken()
     {
-        if (char.IsDigit(current))
+        var lineStart = _line;
+        var columnStart = _column;
+
+        if (char.IsWhiteSpace(_content[_index]))
+        {
+            while (_index < _content.Length && char.IsWhiteSpace(_content[_index]))
+            {
+                Next();
+            }
+
+            return new WhitespaceToken(CreateSpan(lineStart, columnStart));
+        }
+
+        if (_content[_index] == '/' && _index + 1 < _content.Length && _content[_index + 1] == '/')
+        {
+            var startIndex = _index;
+
+            Next(2);
+            while (_index < _content.Length && _content[_index] != '\n')
+            {
+                Next();
+            }
+
+            return new CommentToken(CreateSpan(lineStart, columnStart), _content.AsSpan(startIndex, _index - startIndex).ToString());
+        }
+
+        if (char.IsDigit(_content[_index]))
         {
             return ParseNumber(lineStart, columnStart);
         }
 
-        if (current == '"')
+        if (_content[_index] == '"')
         {
             return ParseString(lineStart, columnStart);
         }
@@ -87,12 +88,12 @@ public sealed class Tokenizer
             }
         }
 
-        if (char.IsLetter(current) || current == '_')
+        if (char.IsLetter(_content[_index]) || _content[_index] == '_')
         {
             return ParseIdentifier(lineStart, columnStart);
         }
 
-        throw new CompileException(Diagnostic.Error($"Unknown token '{current}'").Build());
+        throw new CompileException(Diagnostic.Error($"Unknown token '{_content[_index]}'").Build());
     }
 
     private Token ParseNumber(int lineStart, int columnStart)
@@ -382,7 +383,26 @@ public sealed class Tokenizer
 
     private void Next(int count = 1)
     {
-        _index += count;
-        _column += count;
+        for (var i = 0; i < count; i++)
+        {
+            if (_index < _content.Length)
+            {
+                if (_content[_index] == '\n')
+                {
+                    _line += 1;
+                    _column = 1;
+                }
+                else
+                {
+                    _column++;
+                }
+            }
+            else
+            {
+                _column++;
+            }
+
+            _index++;
+        }
     }
 }