diff --git a/example/program.nub b/example/program.nub index 2da8f81..6a1e761 100644 --- a/example/program.nub +++ b/example/program.nub @@ -9,7 +9,7 @@ struct Human { export func main(args: []^string) { let i: i64 - c::printf("%d\n", args.count) + c:printf("%d\n", args.count) while i < args.count { c::printf("%s\n", args[i]) diff --git a/src/lang/Nub.Lang.CLI/Program.cs b/src/lang/Nub.Lang.CLI/Program.cs index fefced7..8d1efda 100644 --- a/src/lang/Nub.Lang.CLI/Program.cs +++ b/src/lang/Nub.Lang.CLI/Program.cs @@ -1,4 +1,5 @@ -using Nub.Lang.Backend; +using Nub.Lang; +using Nub.Lang.Backend; using Nub.Lang.Frontend.Diagnostics; using Nub.Lang.Frontend.Lexing; using Nub.Lang.Frontend.Parsing; @@ -29,7 +30,7 @@ foreach (var file in Directory.EnumerateFiles(srcDir, "*.nub", SearchOption.AllD { var content = File.ReadAllText(file); - var tokenizeResult = lexer.Tokenize(new SourceText(file, content)); + var tokenizeResult = lexer.Tokenize(new SourceText(content)); tokenizeResult.PrintAllDiagnostics(); error = error || tokenizeResult.HasErrors; diff --git a/src/lang/Nub.Lang/Frontend/Diagnostics/ConsoleColors.cs b/src/lang/Nub.Lang/Frontend/Diagnostics/ConsoleColors.cs index 2222701..555af8c 100644 --- a/src/lang/Nub.Lang/Frontend/Diagnostics/ConsoleColors.cs +++ b/src/lang/Nub.Lang/Frontend/Diagnostics/ConsoleColors.cs @@ -13,7 +13,7 @@ public static class ConsoleColors public const string BrightWhite = "\e[97m"; public const string Gray = "\e[90m"; - public static bool IsColorSupported() + private static bool IsColorSupported() { var term = Environment.GetEnvironmentVariable("TERM"); var colorTerm = Environment.GetEnvironmentVariable("COLORTERM"); diff --git a/src/lang/Nub.Lang/Frontend/Diagnostics/Diagnostic.cs b/src/lang/Nub.Lang/Frontend/Diagnostics/Diagnostic.cs index 1606084..be9587f 100644 --- a/src/lang/Nub.Lang/Frontend/Diagnostics/Diagnostic.cs +++ b/src/lang/Nub.Lang/Frontend/Diagnostics/Diagnostic.cs @@ -4,22 +4,14 @@ using Nub.Lang.Frontend.Parsing; namespace Nub.Lang.Frontend.Diagnostics; -public enum DiagnosticSeverity -{ - Info, - Warning, - Error -} - public class Diagnostic { public class DiagnosticBuilder { private readonly DiagnosticSeverity _severity; private readonly string _message; - private SourceText? _sourceFile; - private SourceSpan? _span; private string? _help; + private SourceSpan? _sourceSpan; public DiagnosticBuilder(DiagnosticSeverity severity, string message) { @@ -29,27 +21,19 @@ public class Diagnostic public DiagnosticBuilder At(Token token) { - _sourceFile = token.SourceText; - _span = SourceLocationCalculator.GetSpan(token); + _sourceSpan = token.Span; return this; } public DiagnosticBuilder At(Node node) { - if (!node.Tokens.Any()) - { - throw new ArgumentException("Node has no tokens", nameof(node)); - } - - _sourceFile = node.Tokens[0].SourceText; - _span = SourceLocationCalculator.GetSpan(node); + _sourceSpan = SourceSpan.Merge(node.Tokens.Select(t => t.Span)); return this; } - public DiagnosticBuilder At(SourceText sourceText, SourceSpan span) + public DiagnosticBuilder At(SourceSpan span) { - _sourceFile = sourceText; - _span = span; + _sourceSpan = span; return this; } @@ -59,7 +43,7 @@ public class Diagnostic return this; } - public Diagnostic Build() => new(_severity, _message, _sourceFile, _span, _help); + public Diagnostic Build() => new(_severity, _message, _sourceSpan, _help); } public static DiagnosticBuilder Error(string message) => new(DiagnosticSeverity.Error, message); @@ -68,15 +52,13 @@ public class Diagnostic public DiagnosticSeverity Severity { get; } public string Message { get; } - public SourceText? SourceFile { get; } public SourceSpan? Span { get; } public string? Help { get; } - private Diagnostic(DiagnosticSeverity severity, string message, SourceText? sourceFile, SourceSpan? span, string? help) + private Diagnostic(DiagnosticSeverity severity, string message, SourceSpan? span, string? help) { Severity = severity; Message = message; - SourceFile = sourceFile; Span = span; Help = help; } @@ -88,18 +70,18 @@ public class Diagnostic var severityText = GetSeverityText(Severity); sb.Append(severityText); - if (SourceFile.HasValue) + if (Span.HasValue) { - var locationText = $" at {SourceFile.Value.Path}:{Span}"; - sb.Append(ConsoleColors.Colorize(locationText, ConsoleColors.Gray)); + // var locationText = $" at {Span.Value.Path}:{Span}"; + // sb.Append(ConsoleColors.Colorize(locationText, ConsoleColors.Gray)); } sb.Append(": "); sb.AppendLine(ConsoleColors.Colorize(Message, ConsoleColors.BrightWhite)); - if (SourceFile.HasValue && Span.HasValue) + if (Span.HasValue) { - AppendSourceContext(sb, SourceFile.Value, Span.Value); + AppendSourceContext(sb, Span.Value); } if (!string.IsNullOrEmpty(Help)) @@ -123,9 +105,9 @@ public class Diagnostic }; } - private static void AppendSourceContext(StringBuilder sb, SourceText sourceText, SourceSpan span) + private static void AppendSourceContext(StringBuilder sb, SourceSpan span) { - var lines = sourceText.Content.Split('\n'); + var lines = span.Content.Text.Split('\n'); var startLine = span.Start.Line; var endLine = span.End.Line; @@ -143,7 +125,8 @@ public class Diagnostic for (var lineNum = contextStart; lineNum < startLine; lineNum++) { - AppendContextLine(sb, lineNum, lines[lineNum - 1], lineNumWidth); + var line = lines[lineNum - 1]; + AppendContextLine(sb, lineNum, line, lineNumWidth); } for (var lineNum = startLine; lineNum <= endLine; lineNum++) @@ -156,7 +139,8 @@ public class Diagnostic var contextEnd = Math.Min(lines.Length, endLine + contextLines); for (var lineNum = endLine + 1; lineNum <= contextEnd; lineNum++) { - AppendContextLine(sb, lineNum, lines[lineNum - 1], lineNumWidth); + var line = lines[lineNum - 1]; + AppendContextLine(sb, lineNum, line, lineNumWidth); } } @@ -202,4 +186,11 @@ public class Diagnostic return new string(indicator, line.Length); } +} + +public enum DiagnosticSeverity +{ + Info, + Warning, + Error } \ No newline at end of file diff --git a/src/lang/Nub.Lang/Frontend/Diagnostics/SourceText.cs b/src/lang/Nub.Lang/Frontend/Diagnostics/SourceText.cs deleted file mode 100644 index c41b2f9..0000000 --- a/src/lang/Nub.Lang/Frontend/Diagnostics/SourceText.cs +++ /dev/null @@ -1,102 +0,0 @@ -using Nub.Lang.Frontend.Lexing; -using Nub.Lang.Frontend.Parsing; - -namespace Nub.Lang.Frontend.Diagnostics; - -public readonly struct SourceText(string path, string content) -{ - public string Path { get; } = path; - public string Content { get; } = content; -} - -public readonly struct SourceLocation(int line, int column) -{ - public int Line { get; } = line; - public int Column { get; } = column; -} - -public readonly struct SourceSpan(SourceLocation start, SourceLocation end) -{ - public SourceLocation Start { get; } = start; - public SourceLocation End { get; } = end; - - public override string ToString() - { - if (Start.Line == End.Line) - { - if (Start.Column == End.Column) - { - return $"{Start.Line}:{Start.Column}"; - } - - return $"{Start.Line}:{Start.Column}-{End.Column}"; - } - - return $"{Start.Line}:{Start.Column}-{End.Line}:{End.Column}"; - } -} - -public static class SourceLocationCalculator -{ - private static int[] GetLineStarts(string content) - { - var lineStarts = new List { 0 }; - - for (var i = 0; i < content.Length; i++) - { - if (content[i] == '\n') - { - lineStarts.Add(i + 1); - } - } - - return lineStarts.ToArray(); - } - - private static SourceLocation IndexToLocation(string content, int index) - { - if (index < 0 || index > content.Length) - { - throw new ArgumentOutOfRangeException(nameof(index), $"Index {index} is out of range for content of length {content.Length}"); - } - - var lineStarts = GetLineStarts(content); - - var line = Array.BinarySearch(lineStarts, index); - if (line < 0) - { - line = ~line - 1; - } - - if (line < lineStarts.Length - 1 && lineStarts[line + 1] == index && index < content.Length && content[index] == '\n') - { - line++; - } - - var column = index - lineStarts[line] + 1; - return new SourceLocation(line + 1, column); - } - - public static SourceSpan GetSpan(Token token) - { - var start = IndexToLocation(token.SourceText.Content, token.StartIndex); - var end = IndexToLocation(token.SourceText.Content, token.EndIndex); - return new SourceSpan(start, end); - } - - public static SourceSpan GetSpan(Node node) - { - if (!node.Tokens.Any()) - { - throw new ArgumentException("Node has no tokens", nameof(node)); - } - - var firstToken = node.Tokens[0]; - var lastToken = node.Tokens[^1]; - - var start = IndexToLocation(firstToken.SourceText.Content, firstToken.StartIndex); - var end = IndexToLocation(lastToken.SourceText.Content, lastToken.EndIndex); - - return new SourceSpan(start, end); - } -} \ No newline at end of file diff --git a/src/lang/Nub.Lang/Frontend/Lexing/DocumentationToken.cs b/src/lang/Nub.Lang/Frontend/Lexing/DocumentationToken.cs index 1dcbe3f..139494b 100644 --- a/src/lang/Nub.Lang/Frontend/Lexing/DocumentationToken.cs +++ b/src/lang/Nub.Lang/Frontend/Lexing/DocumentationToken.cs @@ -2,7 +2,7 @@ using Nub.Lang.Frontend.Diagnostics; namespace Nub.Lang.Frontend.Lexing; -public class DocumentationToken(SourceText sourceText, int startIndex, int endIndex, string documentation) : Token(sourceText, startIndex, endIndex) +public class DocumentationToken(SourceSpan span, string documentation) : Token(span) { public string Documentation { get; } = documentation; } \ No newline at end of file diff --git a/src/lang/Nub.Lang/Frontend/Lexing/IdentifierToken.cs b/src/lang/Nub.Lang/Frontend/Lexing/IdentifierToken.cs index 5494f72..62b4664 100644 --- a/src/lang/Nub.Lang/Frontend/Lexing/IdentifierToken.cs +++ b/src/lang/Nub.Lang/Frontend/Lexing/IdentifierToken.cs @@ -2,7 +2,7 @@ namespace Nub.Lang.Frontend.Lexing; -public class IdentifierToken(SourceText sourceText, int startIndex, int endIndex, string value) : Token(sourceText, startIndex, endIndex) +public class IdentifierToken(SourceSpan span, string value) : Token(span) { public string Value { get; } = value; } \ No newline at end of file diff --git a/src/lang/Nub.Lang/Frontend/Lexing/Lexer.cs b/src/lang/Nub.Lang/Frontend/Lexing/Lexer.cs index f3187ab..ce3c76d 100644 --- a/src/lang/Nub.Lang/Frontend/Lexing/Lexer.cs +++ b/src/lang/Nub.Lang/Frontend/Lexing/Lexer.cs @@ -109,7 +109,7 @@ public class Lexer Next(); } Next(); - return new DocumentationToken(_sourceText, startIndex, _index, buffer); + return new DocumentationToken(CreateSpan(startIndex), buffer); } while (Peek().TryGetValue(out var character) && character != '\n') @@ -132,20 +132,20 @@ public class Lexer if (Keywords.TryGetValue(buffer, out var keywordSymbol)) { - return new SymbolToken(_sourceText, startIndex, _index, keywordSymbol); + return new SymbolToken(CreateSpan(startIndex), keywordSymbol); } if (Modifiers.TryGetValue(buffer, out var modifer)) { - return new ModifierToken(_sourceText, startIndex, _index, modifer); + return new ModifierToken(CreateSpan(startIndex), modifer); } if (buffer is "true" or "false") { - return new LiteralToken(_sourceText, startIndex, _index, NubPrimitiveType.Bool, buffer); + return new LiteralToken(CreateSpan(startIndex), NubPrimitiveType.Bool, buffer); } - return new IdentifierToken(_sourceText, startIndex, _index, buffer); + return new IdentifierToken(CreateSpan(startIndex), buffer); } if (char.IsDigit(current)) @@ -183,7 +183,7 @@ public class Lexer } } - return new LiteralToken(_sourceText, startIndex, _index, isFloat ? NubPrimitiveType.F64 : NubPrimitiveType.I64, buffer); + return new LiteralToken(CreateSpan(startIndex), isFloat ? NubPrimitiveType.F64 : NubPrimitiveType.I64, buffer); } // TODO: Revisit this @@ -203,7 +203,7 @@ public class Lexer Next(); } - return new SymbolToken(_sourceText, startIndex, _index, chain.Value); + return new SymbolToken(CreateSpan(startIndex), chain.Value); } } } @@ -211,7 +211,7 @@ public class Lexer if (Chars.TryGetValue(current, out var charSymbol)) { Next(); - return new SymbolToken(_sourceText, startIndex, _index, charSymbol); + return new SymbolToken(CreateSpan(startIndex), charSymbol); } if (current == '"') @@ -236,17 +236,42 @@ public class Lexer Next(); } - return new LiteralToken(_sourceText, startIndex, _index, NubPrimitiveType.String, buffer); + return new LiteralToken(CreateSpan(startIndex), NubPrimitiveType.String, buffer); } throw new Exception($"Unknown character {current}"); } + private SourceLocation CreateLocation(int index) + { + var line = 1; + var column = 0; + for (var i = 0; i < index; i++) + { + if (_sourceText.Text[i] == '\n') + { + column = 1; + line += 1; + } + else + { + column += 1; + } + } + + return new SourceLocation(line, column); + } + + private SourceSpan CreateSpan(int startIndex) + { + return new SourceSpan(_sourceText, CreateLocation(startIndex), CreateLocation(_index)); + } + private Optional Peek(int offset = 0) { - if (_index + offset < _sourceText.Content.Length) + if (_index + offset < _sourceText.Text.Length) { - return _sourceText.Content[_index + offset]; + return _sourceText.Text[_index + offset]; } return Optional.Empty(); diff --git a/src/lang/Nub.Lang/Frontend/Lexing/LiteralToken.cs b/src/lang/Nub.Lang/Frontend/Lexing/LiteralToken.cs index 8d45d84..07b236f 100644 --- a/src/lang/Nub.Lang/Frontend/Lexing/LiteralToken.cs +++ b/src/lang/Nub.Lang/Frontend/Lexing/LiteralToken.cs @@ -3,7 +3,7 @@ using Nub.Lang.Frontend.Typing; namespace Nub.Lang.Frontend.Lexing; -public class LiteralToken(SourceText sourceText, int startIndex, int endIndex, NubType type, string value) : Token(sourceText, startIndex, endIndex) +public class LiteralToken(SourceSpan span, NubType type, string value) : Token(span) { public NubType Type { get; } = type; public string Value { get; } = value; diff --git a/src/lang/Nub.Lang/Frontend/Lexing/ModifierToken.cs b/src/lang/Nub.Lang/Frontend/Lexing/ModifierToken.cs index 4170ea8..cf9b890 100644 --- a/src/lang/Nub.Lang/Frontend/Lexing/ModifierToken.cs +++ b/src/lang/Nub.Lang/Frontend/Lexing/ModifierToken.cs @@ -2,7 +2,7 @@ using Nub.Lang.Frontend.Diagnostics; namespace Nub.Lang.Frontend.Lexing; -public class ModifierToken(SourceText sourceText, int startIndex, int endIndex, Modifier modifier) : Token(sourceText, startIndex, endIndex) +public class ModifierToken(SourceSpan span, Modifier modifier) : Token(span) { public Modifier Modifier { get; } = modifier; } diff --git a/src/lang/Nub.Lang/Frontend/Lexing/SymbolToken.cs b/src/lang/Nub.Lang/Frontend/Lexing/SymbolToken.cs index b01a0fe..de3314e 100644 --- a/src/lang/Nub.Lang/Frontend/Lexing/SymbolToken.cs +++ b/src/lang/Nub.Lang/Frontend/Lexing/SymbolToken.cs @@ -2,7 +2,7 @@ namespace Nub.Lang.Frontend.Lexing; -public class SymbolToken(SourceText sourceText, int startIndex, int endIndex, Symbol symbol) : Token(sourceText, startIndex, endIndex) +public class SymbolToken(SourceSpan span, Symbol symbol) : Token(span) { public Symbol Symbol { get; } = symbol; } diff --git a/src/lang/Nub.Lang/Frontend/Lexing/Token.cs b/src/lang/Nub.Lang/Frontend/Lexing/Token.cs index 1a46ae8..05d903f 100644 --- a/src/lang/Nub.Lang/Frontend/Lexing/Token.cs +++ b/src/lang/Nub.Lang/Frontend/Lexing/Token.cs @@ -2,9 +2,7 @@ namespace Nub.Lang.Frontend.Lexing; -public abstract class Token(SourceText sourceText, int startIndex, int endIndex) +public abstract class Token(SourceSpan span) { - public SourceText SourceText { get; } = sourceText; - public int StartIndex { get; } = startIndex; - public int EndIndex { get; } = endIndex; + public SourceSpan Span { get; } = span; } \ No newline at end of file diff --git a/src/lang/Nub.Lang/Frontend/Parsing/Parser.cs b/src/lang/Nub.Lang/Frontend/Parsing/Parser.cs index 3ca3a96..2a62f03 100644 --- a/src/lang/Nub.Lang/Frontend/Parsing/Parser.cs +++ b/src/lang/Nub.Lang/Frontend/Parsing/Parser.cs @@ -672,7 +672,7 @@ public class Parser throw new ParseException(Diagnostic .Error("Unexpected end of file while parsing type") .WithHelp("Expected a type name") - .At(_tokens.Last().SourceText, SourceLocationCalculator.GetSpan(_tokens.Last())) + .At(_tokens.Last()) .Build()); } @@ -690,7 +690,7 @@ public class Parser throw new ParseException(Diagnostic .Error("Unexpected end of file") .WithHelp("Expected more tokens to complete the syntax") - .At(_tokens.Last().SourceText, SourceLocationCalculator.GetSpan(_tokens.Last())) + .At(_tokens.Last()) .Build()); } diff --git a/src/lang/Nub.Lang/Source.cs b/src/lang/Nub.Lang/Source.cs new file mode 100644 index 0000000..2658c75 --- /dev/null +++ b/src/lang/Nub.Lang/Source.cs @@ -0,0 +1,136 @@ +using System.Diagnostics.CodeAnalysis; + +namespace Nub.Lang; + +public readonly struct SourceSpan(SourceText content, SourceLocation start, SourceLocation end) : IEquatable +{ + public SourceText Content { get; } = content; + public SourceLocation Start { get; } = start; + public SourceLocation End { get; } = end; + + /// + /// Merges one or more from a single file to a single + /// + /// The spans to merged + /// The merged + public static SourceSpan Merge(IEnumerable spanEnumerable) + { + var spans = spanEnumerable.ToArray(); + if (spans.Length == 0) + { + throw new ArgumentException("Cannot merge empty spans", nameof(spanEnumerable)); + } + + var files = spans.Select(s => s.Content).Distinct().ToArray(); + if (files.Length > 1) + { + throw new ArgumentException("Cannot merge spans from multiple files", nameof(spanEnumerable)); + } + + var first = spans.OrderBy(s => s.Start.Line).ThenBy(s => s.Start.Column).First().Start; + var last = spans.OrderByDescending(s => s.End.Line).ThenByDescending(s => s.End.Column).Last().End; + + return new SourceSpan(files[0], first, last); + } + + public override string ToString() + { + if (Start.Line == End.Line) + { + if (Start.Column == End.Column) + { + return $"{Start.Line}:{Start.Column}"; + } + + return $"{Start.Line}:{Start.Column}-{End.Column}"; + } + + return $"{Start.Line}:{Start.Column}-{End.Line}:{End.Column}"; + } + + public override bool Equals([NotNullWhen(true)] object? obj) + { + return obj is SourceSpan other && Equals(other); + } + + public bool Equals(SourceSpan other) + { + return Content.Equals(other.Content) && Start.Equals(other.Start) && End.Equals(other.End); + } + + public override int GetHashCode() + { + return HashCode.Combine(Content, Start, End); + } + + public static bool operator ==(SourceSpan left, SourceSpan right) + { + return left.Equals(right); + } + + public static bool operator !=(SourceSpan left, SourceSpan right) + { + return !(left == right); + } +} + +public readonly struct SourceText(string text) : IEquatable +{ + public string Text { get; } = text; + + public bool Equals(SourceText other) + { + return Text == other.Text; + } + + public override bool Equals([NotNullWhen(true)] object? obj) + { + return obj is SourceText other && Equals(other); + } + + public override int GetHashCode() + { + return Text.GetHashCode(); + } + + public static bool operator ==(SourceText left, SourceText right) + { + return left.Equals(right); + } + + public static bool operator !=(SourceText left, SourceText right) + { + return !(left == right); + } +} + +public readonly struct SourceLocation(int line, int column) : IEquatable +{ + public int Line { get; } = line; + public int Column { get; } = column; + + public override bool Equals([NotNullWhen(true)] object? obj) + { + return obj is SourceLocation other && Equals(other); + } + + public bool Equals(SourceLocation other) + { + return Line == other.Line && Column == other.Column; + } + + public override int GetHashCode() + { + return HashCode.Combine(Line, Column); + } + + public static bool operator ==(SourceLocation left, SourceLocation right) + { + return left.Equals(right); + } + + public static bool operator !=(SourceLocation left, SourceLocation right) + { + return !(left == right); + } +} \ No newline at end of file