...

src/compiler/NubLang/Tokenization/Tokenizer.cs

@@ -0,0 +1,224 @@
+using NubLang.Common;
+
+namespace NubLang.Tokenization;
+
+public sealed class Tokenizer
+{
+    private static readonly Dictionary<string, Symbol> Keywords = new()
+    {
+        ["func"] = Symbol.Func,
+        ["if"] = Symbol.If,
+        ["else"] = Symbol.Else,
+        ["while"] = Symbol.While,
+        ["break"] = Symbol.Break,
+        ["continue"] = Symbol.Continue,
+        ["return"] = Symbol.Return,
+        ["alloc"] = Symbol.Alloc,
+        ["struct"] = Symbol.Struct,
+        ["let"] = Symbol.Let,
+        ["calls"] = Symbol.Calls,
+        ["interface"] = Symbol.Interface,
+        ["for"] = Symbol.For,
+        ["extern"] = Symbol.Extern,
+    };
+
+    private static readonly Dictionary<char[], Symbol> Symbols = new()
+    {
+        [['=', '=']] = Symbol.Equal,
+        [['!', '=']] = Symbol.NotEqual,
+        [['<', '=']] = Symbol.LessThanOrEqual,
+        [['>', '=']] = Symbol.GreaterThanOrEqual,
+        [['=', '>']] = Symbol.Arrow,
+        [[':']] = Symbol.Colon,
+        [['(']] = Symbol.OpenParen,
+        [[')']] = Symbol.CloseParen,
+        [['{']] = Symbol.OpenBrace,
+        [['}']] = Symbol.CloseBrace,
+        [['[']] = Symbol.OpenBracket,
+        [[']']] = Symbol.CloseBracket,
+        [[',']] = Symbol.Comma,
+        [['.']] = Symbol.Period,
+        [['=']] = Symbol.Assign,
+        [['<']] = Symbol.LessThan,
+        [['>']] = Symbol.GreaterThan,
+        [['+']] = Symbol.Plus,
+        [['-']] = Symbol.Minus,
+        [['*']] = Symbol.Star,
+        [['/']] = Symbol.ForwardSlash,
+        [['!']] = Symbol.Bang,
+        [['^']] = Symbol.Caret,
+        [['&']] = Symbol.Ampersand,
+        [[';']] = Symbol.Semi,
+    };
+
+    private static readonly (char[] Pattern, Symbol Symbol)[] OrderedSymbols = Symbols
+        .OrderByDescending(kvp => kvp.Key.Length)
+        .Select(kvp => (kvp.Key, kvp.Value))
+        .ToArray();
+
+    private readonly string _sourceText;
+    private int _index;
+
+    public Tokenizer(string sourceText)
+    {
+        _sourceText = sourceText;
+    }
+
+    public IEnumerable<Token> Tokenize()
+    {
+        _index = 0;
+
+        while (Peek().TryGetValue(out var current))
+        {
+            if (char.IsWhiteSpace(current))
+            {
+                Next();
+                continue;
+            }
+
+            if (current == '/' && Peek(1).TryGetValue(out var nextChar) && nextChar == '/')
+            {
+                while (Peek().TryGetValue(out var ch) && ch != '\n')
+                {
+                    Next();
+                }
+
+                continue;
+            }
+
+            if (char.IsLetter(current) || current == '_')
+            {
+                var buffer = string.Empty;
+
+                while (Peek().TryGetValue(out var next) && (char.IsLetterOrDigit(next) || next == '_'))
+                {
+                    buffer += next;
+                    Next();
+                }
+
+                if (Keywords.TryGetValue(buffer, out var keywordSymbol))
+                {
+                    yield return new SymbolToken(keywordSymbol);
+                    continue;
+                }
+
+                if (buffer is "true" or "false")
+                {
+                    yield return new LiteralToken(LiteralKind.Bool, buffer);
+                    continue;
+                }
+
+                yield return new IdentifierToken(buffer);
+                continue;
+            }
+
+            if (char.IsDigit(current))
+            {
+                var isFloat = false;
+                var buffer = string.Empty;
+
+                while (Peek().TryGetValue(out var next))
+                {
+                    if (next == '.')
+                    {
+                        if (isFloat)
+                        {
+                            throw new Exception("More than one period found in float literal");
+                        }
+
+                        isFloat = true;
+                        buffer += next;
+                        Next();
+                    }
+                    else if (char.IsDigit(next))
+                    {
+                        buffer += next;
+                        Next();
+                    }
+                    else
+                    {
+                        break;
+                    }
+                }
+
+                yield return new LiteralToken(isFloat ? LiteralKind.Float : LiteralKind.Integer, buffer);
+                continue;
+            }
+
+            if (current == '"')
+            {
+                Next();
+                var buffer = string.Empty;
+
+                while (true)
+                {
+                    if (!Peek().TryGetValue(out var next))
+                    {
+                        throw new Exception("Unclosed string literal");
+                    }
+
+                    if (next == '"')
+                    {
+                        Next();
+                        break;
+                    }
+
+                    buffer += next;
+                    Next();
+                }
+
+                yield return new LiteralToken(LiteralKind.String, buffer);
+                continue;
+            }
+
+            var foundMatch = false;
+            foreach (var (pattern, symbol) in OrderedSymbols)
+            {
+                for (var i = 0; i < pattern.Length; i++)
+                {
+                    var c = Peek(i);
+                    if (!c.HasValue || c.Value != pattern[i]) break;
+
+                    if (i == pattern.Length - 1)
+                    {
+                        for (var j = 0; j <= i; j++)
+                        {
+                            Next();
+                        }
+
+                        yield return new SymbolToken(symbol);
+                        foundMatch = true;
+                        break;
+                    }
+                }
+
+                if (foundMatch)
+                {
+                    break;
+                }
+            }
+
+            if (foundMatch)
+            {
+                continue;
+            }
+
+            throw new Exception($"Unknown character {current}");
+        }
+    }
+
+    private Optional<char> Peek(int offset = 0)
+    {
+        if (_index + offset < _sourceText.Length)
+        {
+            return _sourceText[_index + offset];
+        }
+
+        return Optional<char>.Empty();
+    }
+
+    private void Next()
+    {
+        _index++;
+    }
+}