...

compiler/NubLang/Syntax/Tokenizer.cs

@@ -68,19 +68,17 @@ public sealed class Tokenizer
 
     private Token ParseToken(char current, int lineStart, int columnStart)
     {
-        // Numbers
         if (char.IsDigit(current))
         {
             return ParseNumber(lineStart, columnStart);
         }
 
-        // String literals
         if (current == '"')
         {
             return ParseString(lineStart, columnStart);
         }
 
-        // Try keywords and symbols by length (longest first)
+        // note(nub31): Look for keywords (longest first in case a keyword fits partially in a larger keyword)
         for (var i = 8; i >= 1; i--)
         {
             if (TryMatchSymbol(i, lineStart, columnStart, out var token))
@@ -89,7 +87,6 @@ public sealed class Tokenizer
             }
         }
 
-        // Identifiers
         if (char.IsLetter(current) || current == '_')
         {
             return ParseIdentifier(lineStart, columnStart);
@@ -103,7 +100,7 @@ public sealed class Tokenizer
         var start = _index;
         var current = _content[_index];
 
-        // Hex literal
+        // note(nub31): 0xFFFFFF
         if (current == '0' && _index + 1 < _content.Length && _content[_index + 1] == 'x')
         {
             Next(2);
@@ -128,7 +125,7 @@ public sealed class Tokenizer
                 16);
         }
 
-        // Binary literal
+        // note(nub31): 0b11001100
         if (current == '0' && _index + 1 < _content.Length && _content[_index + 1] == 'b')
         {
             Next(2);
@@ -153,7 +150,7 @@ public sealed class Tokenizer
                 2);
         }
 
-        // Decimal or float
+        // note(nub31): 23/23.5
         var isFloat = false;
         while (_index < _content.Length)
         {
@@ -191,7 +188,7 @@ public sealed class Tokenizer
 
     private StringLiteralToken ParseString(int lineStart, int columnStart)
     {
-        Next(); // Skip opening quote
+        Next();
         var start = _index;
 
         while (true)
@@ -236,6 +233,20 @@ public sealed class Tokenizer
 
         var span = _content.AsSpan(_index, length);
 
+        if (span is "true")
+        {
+            Next(4);
+            token = new BoolLiteralToken(CreateSpan(lineStart, columnStart), true);
+            return true;
+        }
+
+        if (span is "false")
+        {
+            Next(5);
+            token = new BoolLiteralToken(CreateSpan(lineStart, columnStart), false);
+            return true;
+        }
+
         var symbol = length switch
         {
             8 => span switch
@@ -287,6 +298,7 @@ public sealed class Tokenizer
                 "&&" => Symbol.And,
                 "||" => Symbol.Or,
                 "::" => Symbol.DoubleColon,
+                "x|" => Symbol.XOr,
                 _ => Symbol.None
             },
             1 => span[0] switch
@@ -361,9 +373,7 @@ public sealed class Tokenizer
             }
         }
 
-        return new IdentifierToken(
-            CreateSpan(lineStart, columnStart),
-            _content.Substring(start, _index - start));
+        return new IdentifierToken(CreateSpan(lineStart, columnStart), _content.Substring(start, _index - start));
     }
 
     private SourceSpan CreateSpan(int lineStart, int columnStart)