Diagnostics

src/compiler/NubLang/Tokenization/Tokenizer.cs

@@ -1,4 +1,7 @@
-namespace NubLang.Tokenization;
+using NubLang.Code;
+using NubLang.Diagnostics;
+
+namespace NubLang.Tokenization;
 
 public sealed class Tokenizer
 {
@@ -55,6 +58,8 @@ public sealed class Tokenizer
         .ToArray();
 
     private readonly string _sourceText;
+    private readonly SourceFile? _sourceFile;
+    private readonly List<Diagnostic> _diagnostics = [];
     private int _index;
 
     public Tokenizer(string sourceText)
@@ -62,6 +67,14 @@ public sealed class Tokenizer
         _sourceText = sourceText;
     }
 
+    public Tokenizer(SourceFile sourceFile)
+    {
+        _sourceFile = sourceFile;
+        _sourceText = sourceFile.GetText();
+    }
+
+    public IReadOnlyList<Diagnostic> GetDiagnostics() => _diagnostics;
+
     public IEnumerable<Token> Tokenize()
     {
         _index = 0;
@@ -84,6 +97,8 @@ public sealed class Tokenizer
                 continue;
             }
 
+            var tokenStartIndex = _index;
+
             if (char.IsLetter(current) || current == '_')
             {
                 var buffer = string.Empty;
@@ -96,17 +111,17 @@ public sealed class Tokenizer
 
                 if (Keywords.TryGetValue(buffer, out var keywordSymbol))
                 {
-                    yield return new SymbolToken(keywordSymbol);
+                    yield return new SymbolToken(GetSourceFileSpan(tokenStartIndex), keywordSymbol);
                     continue;
                 }
 
                 if (buffer is "true" or "false")
                 {
-                    yield return new LiteralToken(LiteralKind.Bool, buffer);
+                    yield return new LiteralToken(GetSourceFileSpan(tokenStartIndex), LiteralKind.Bool, buffer);
                     continue;
                 }
 
-                yield return new IdentifierToken(buffer);
+                yield return new IdentifierToken(GetSourceFileSpan(tokenStartIndex), buffer);
                 continue;
             }
 
@@ -139,7 +154,7 @@ public sealed class Tokenizer
                     }
                 }
 
-                yield return new LiteralToken(isFloat ? LiteralKind.Float : LiteralKind.Integer, buffer);
+                yield return new LiteralToken(GetSourceFileSpan(tokenStartIndex), isFloat ? LiteralKind.Float : LiteralKind.Integer, buffer);
                 continue;
             }
 
@@ -165,7 +180,7 @@ public sealed class Tokenizer
                     Next();
                 }
 
-                yield return new LiteralToken(LiteralKind.String, buffer);
+                yield return new LiteralToken(GetSourceFileSpan(tokenStartIndex), LiteralKind.String, buffer);
                 continue;
             }
 
@@ -184,7 +199,7 @@ public sealed class Tokenizer
                             Next();
                         }
 
-                        yield return new SymbolToken(symbol);
+                        yield return new SymbolToken(GetSourceFileSpan(tokenStartIndex), symbol);
                         foundMatch = true;
                         break;
                     }
@@ -201,7 +216,8 @@ public sealed class Tokenizer
                 continue;
             }
 
-            throw new Exception($"Unknown character {current}");
+            _diagnostics.Add(Diagnostic.Error($"Unknown token '{current}'").At(GetSourceFileSpan(tokenStartIndex)).Build());
+            Next();
         }
     }
 
@@ -219,4 +235,37 @@ public sealed class Tokenizer
     {
         _index++;
     }
+
+    private SourceFileSpan? GetSourceFileSpan(int tokenStartIndex)
+    {
+        if (_sourceFile != null)
+        {
+            var start = CalculateSourceLocation(tokenStartIndex);
+            var end = CalculateSourceLocation(_index + 1);
+            return new SourceFileSpan(_sourceFile, new SourceSpan(start, end));
+        }
+
+        return null;
+    }
+
+    private SourceLocation CalculateSourceLocation(int index)
+    {
+        var line = 1;
+        var column = 1;
+
+        for (var i = 0; i < index && i < _sourceText.Length; i++)
+        {
+            if (_sourceText[i] == '\n')
+            {
+                line++;
+                column = 1;
+            }
+            else
+            {
+                column++;
+            }
+        }
+
+        return new SourceLocation(line, column);
+    }
 }