...

compiler/NubLang/Syntax/Tokenizer.cs

@@ -11,20 +11,20 @@ public sealed class Tokenizer
     private int _column = 1;
 
     public List<Diagnostic> Diagnostics { get; set; } = new(16);
-    public List<Token> Tokens { get; set; } = new(256);
 
-    public void Tokenize(string fileName, string content)
+    public List<Token> Tokenize(string fileName, string content)
     {
         _fileName = fileName;
         _content = content;
 
         Diagnostics = [];
-        Tokens = [];
 
         _index = 0;
         _line = 1;
         _column = 1;
 
+        var tokens = new List<Token>();
+
         while (_index < _content.Length)
         {
             try
@@ -54,7 +54,7 @@ public sealed class Tokenizer
                     continue;
                 }
 
-                Tokens.Add(ParseToken(current, _line, _column));
+                tokens.Add(ParseToken(current, _line, _column));
             }
             catch (CompileException e)
             {
@@ -62,6 +62,8 @@ public sealed class Tokenizer
                 Next();
             }
         }
+
+        return tokens;
     }
 
     private Token ParseToken(char current, int lineStart, int columnStart)
@@ -295,7 +297,6 @@ public sealed class Tokenizer
                 "&&" => Symbol.And,
                 "||" => Symbol.Or,
                 "::" => Symbol.DoubleColon,
-                "x|" => Symbol.XOr,
                 _ => Symbol.None
             },
             1 => span[0] switch
@@ -324,6 +325,7 @@ public sealed class Tokenizer
                 '|' => Symbol.Pipe,
                 '@' => Symbol.At,
                 '?' => Symbol.QuestionMark,
+                '~' => Symbol.Tilde,
                 _ => Symbol.None
             },
             _ => Symbol.None