nub31/nub-lang
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

docs parsing

Browse Source
This commit is contained in:
nub31
2025-05-24 20:59:26 +02:00
parent 0c70710ab1
commit 1ff434e1c3
8 changed files with 84 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
src/compiler/Nub.Lang/Frontend/Lexing/CommentToken.cs
View File

@@ -1,6 +0,0 @@
namespace Nub.Lang.Frontend.Lexing;
public class CommentToken(string comment) : Token
{
public string Comment { get; } = comment;
}

6
src/compiler/Nub.Lang/Frontend/Lexing/DocumentationToken.cs Normal file
View File

@@ -0,0 +1,6 @@
namespace Nub.Lang.Frontend.Lexing;
public class DocumentationToken(string documentation) : Token
{
public string Documentation { get; } = documentation;
}

78
src/compiler/Nub.Lang/Frontend/Lexing/Lexer.cs
View File

@@ -63,34 +63,78 @@ public class Lexer
_index = 0;
List<Token> tokens = [];
while (Peek().TryGetValue(out var character))
while (ParseToken().TryGetValue(out var token))
{
if (char.IsWhiteSpace(character))
{
Next();
continue;
}
tokens.Add(ParseToken(character));
tokens.Add(token);
}
return tokens;
}
private Token ParseToken(char current)
private void ConsumeWhitespace()
{
if (current == '/' && Peek(1) is { Value: '/' })
while (Peek().TryGetValue(out var character) && char.IsWhiteSpace(character))
{
Next();
}
}
private Optional<Token> ParseToken()
{
ConsumeWhitespace();
string? documentation = null;
while (Peek() is { Value: '/' } && Peek(1) is { Value: '/' })
{
Next();
Next();
var buffer = string.Empty;
while (Peek() is not { Value: '\n' })
{
buffer += Peek().Value;
Next();
}
Next();
return new CommentToken(buffer);
if (Peek() is { Value: '/' })
{
Next();
if (documentation != null)
{
documentation += '\n';
}
while (Peek().TryGetValue(out var character))
{
if (character == '\n')
{
Next();
break;
}
documentation += character;
Next();
}
}
else
{
while (Peek().TryGetValue(out var character))
{
if (character == '\n')
{
Next();
break;
}
Next();
}
}
}
if (documentation != null)
{
return new DocumentationToken(documentation);
}
ConsumeWhitespace();
if (!Peek().TryGetValue(out var current))
{
return Optional<Token>.Empty();
}
if (char.IsLetter(current) || current == '_')

5
src/compiler/Nub.Lang/Frontend/Parsing/DefinitionNode.cs
View File

@@ -1,3 +1,6 @@
namespace Nub.Lang.Frontend.Parsing;
public abstract class DefinitionNode : Node;
public abstract class DefinitionNode(Optional<string> documentation) : Node
{
public Optional<string> Documentation { get; set; } = documentation;
}

3
src/compiler/Nub.Lang/Frontend/Parsing/ExternFuncDefinitionNode.cs
View File

@@ -1,11 +1,10 @@
namespace Nub.Lang.Frontend.Parsing;
public class ExternFuncDefinitionNode(string name, List<FuncParameter> parameters, Optional<NubType> returnType, Optional<string> documentation) : DefinitionNode
public class ExternFuncDefinitionNode(string name, List<FuncParameter> parameters, Optional<NubType> returnType, Optional<string> documentation) : DefinitionNode(documentation)
{
public string Name { get; } = name;
public List<FuncParameter> Parameters { get; } = parameters;
public Optional<NubType> ReturnType { get; } = returnType;
public Optional<string> Documentation { get; set; } = documentation;
public override string ToString() => $"{Name}({string.Join(", ", Parameters.Select(p => p.ToString()))}){(ReturnType.HasValue ? ": " + ReturnType.Value : "")}";
}

3
src/compiler/Nub.Lang/Frontend/Parsing/LocalFuncDefinitionNode.cs
View File

@@ -1,13 +1,12 @@
namespace Nub.Lang.Frontend.Parsing;
public class LocalFuncDefinitionNode(string name, List<FuncParameter> parameters, BlockNode body, Optional<NubType> returnType, bool global, Optional<string> documentation) : DefinitionNode
public class LocalFuncDefinitionNode(string name, List<FuncParameter> parameters, BlockNode body, Optional<NubType> returnType, bool global, Optional<string> documentation) : DefinitionNode(documentation)
{
public string Name { get; } = name;
public List<FuncParameter> Parameters { get; } = parameters;
public BlockNode Body { get; } = body;
public Optional<NubType> ReturnType { get; } = returnType;
public bool Global { get; } = global;
public Optional<string> Documentation { get; set; } = documentation;
public override string ToString() => $"{Name}({string.Join(", ", Parameters.Select(p => p.ToString()))}){(ReturnType.HasValue ? ": " + ReturnType.Value : "")}";
}

8
src/compiler/Nub.Lang/Frontend/Parsing/Parser.cs
View File

@@ -37,9 +37,9 @@ public class Parser
List<Modifier> modifiers = [];
List<string> documentationParts = [];
while (_index < _tokens.Count && _tokens[_index] is CommentToken commentToken)
while (_index < _tokens.Count && _tokens[_index] is DocumentationToken commentToken)
{
documentationParts.Add(commentToken.Comment);
documentationParts.Add(commentToken.Documentation);
_index++;
}
@@ -570,7 +570,7 @@ public class Parser
private Optional<Token> Peek()
{
var peekIndex = _index;
while (peekIndex < _tokens.Count && _tokens[peekIndex] is CommentToken)
while (peekIndex < _tokens.Count && _tokens[peekIndex] is DocumentationToken)
{
peekIndex++;
}
@@ -585,7 +585,7 @@ public class Parser
private void Next()
{
while (_index < _tokens.Count && _tokens[_index] is CommentToken)
while (_index < _tokens.Count && _tokens[_index] is DocumentationToken)
{
_index++;
}

3
src/compiler/Nub.Lang/Frontend/Parsing/StructDefinitionNode.cs
View File

@@ -1,8 +1,7 @@
namespace Nub.Lang.Frontend.Parsing;
public class StructDefinitionNode(string name, List<StructField> fields, Optional<string> documentation) : DefinitionNode
public class StructDefinitionNode(string name, List<StructField> fields, Optional<string> documentation) : DefinitionNode(documentation)
{
public string Name { get; } = name;
public List<StructField> Fields { get; } = fields;
public Optional<string> Documentation { get; set; } = documentation;
}