...

src/compiler/Nub.Lang/Frontend/Lexing/Lexer.cs

@@ -8,6 +8,7 @@ public class Lexer
     {
         ["func"] = Symbol.Func,
         ["import"] = Symbol.Import,
+        ["module"] = Symbol.Module,
         ["if"] = Symbol.If,
         ["else"] = Symbol.Else,
         ["while"] = Symbol.While,
@@ -56,14 +57,12 @@ public class Lexer
         ['&'] = Symbol.Ampersand,
     };
 
-    private string _src = null!;
-    private SourceFile _sourceFile;
+    private SourceText _sourceText;
     private int _index;
     
-    public List<Token> Lex(string src, SourceFile sourceFile)
+    public DiagnosticsResult<List<Token>> Tokenize(SourceText sourceText)
     {
-        _src = src;
-        _sourceFile = sourceFile;
+        _sourceText = sourceText;
         _index = 0;
         
         List<Token> tokens = [];
@@ -72,7 +71,7 @@ public class Lexer
             tokens.Add(token);
         }
 
-        return tokens;
+        return new DiagnosticsResult<List<Token>>([], tokens);
     }
 
     private void ConsumeWhitespace()
@@ -123,7 +122,7 @@ public class Lexer
 
         if (documentation != null)
         {
-            return new DocumentationToken(_sourceFile, startIndex, _index, documentation);
+            return new DocumentationToken(_sourceText, startIndex, _index, documentation);
         }
 
         ConsumeWhitespace();
@@ -146,20 +145,20 @@ public class Lexer
 
             if (Keywords.TryGetValue(buffer, out var keywordSymbol))
             {
-                return new SymbolToken(_sourceFile, startIndex, _index, keywordSymbol);
+                return new SymbolToken(_sourceText, startIndex, _index, keywordSymbol);
             }
 
             if (Modifiers.TryGetValue(buffer, out var modifer))
             {
-                return new ModifierToken(_sourceFile, startIndex, _index, modifer);
+                return new ModifierToken(_sourceText, startIndex, _index, modifer);
             }
 
             if (buffer is "true" or "false")
             {
-                return new LiteralToken(_sourceFile, startIndex, _index, NubPrimitiveType.Bool, buffer);
+                return new LiteralToken(_sourceText, startIndex, _index, NubPrimitiveType.Bool, buffer);
             }
 
-            return new IdentifierToken(_sourceFile, startIndex, _index, buffer);
+            return new IdentifierToken(_sourceText, startIndex, _index, buffer);
         }
 
         if (char.IsDigit(current))
@@ -197,7 +196,7 @@ public class Lexer
                 }
             }
 
-            return new LiteralToken(_sourceFile, startIndex, _index, isFloat ? NubPrimitiveType.F64 : NubPrimitiveType.I64, buffer);
+            return new LiteralToken(_sourceText, startIndex, _index, isFloat ? NubPrimitiveType.F64 : NubPrimitiveType.I64, buffer);
         }
 
         // TODO: Revisit this
@@ -217,7 +216,7 @@ public class Lexer
                         Next();
                     }
 
-                    return new SymbolToken(_sourceFile, startIndex, _index, chain.Value);
+                    return new SymbolToken(_sourceText, startIndex, _index, chain.Value);
                 }
             }
         }
@@ -225,7 +224,7 @@ public class Lexer
         if (Chars.TryGetValue(current, out var charSymbol))
         {
             Next();
-            return new SymbolToken(_sourceFile, startIndex, _index, charSymbol);
+            return new SymbolToken(_sourceText, startIndex, _index, charSymbol);
         }
 
         if (current == '"')
@@ -250,7 +249,7 @@ public class Lexer
                 Next();
             }
 
-            return new LiteralToken(_sourceFile, startIndex, _index, NubPrimitiveType.String, buffer);
+            return new LiteralToken(_sourceText, startIndex, _index, NubPrimitiveType.String, buffer);
         }
 
         throw new Exception($"Unknown character {current}");
@@ -258,9 +257,9 @@ public class Lexer
 
     private Optional<char> Peek(int offset = 0)
     {
-        if (_index + offset < _src.Length)
+        if (_index + offset < _sourceText.Content.Length)
         {
-            return _src[_index + offset];
+            return _sourceText.Content[_index + offset];
         }
 
         return Optional<char>.Empty();