nielsbross/Ink
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add sentinel node to parser.

Browse Source
This commit is contained in:
Niels Bross
2025-09-30 07:35:05 +02:00
parent 9ec1b9cf82
commit 344d6b2d31
3 changed files with 65 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
ast.jai
View File

@@ -4,6 +4,7 @@
// [ ] Add a way to infer or get file path from a node
AST_Kind :: enum {
Invalid;
Program;
Function;
Return;

1
module.jai
View File

@@ -161,7 +161,6 @@ Compiler_Context :: struct {
codegen_result_text : string;
typed_buffers : Static_Array(Type_Variable_Handle, 32);
// structured_buffers : Static_Array(Type_Variable_Handle, 16);
scope_stack : Scope_Stack;
type_variables : [..]Type_Variable;

122
parsing.jai
View File

@@ -1,6 +1,6 @@
////////////////////////////
//@nb - Parse_state state
Parse_State :: struct {
Parser :: struct {
current : *Token;
previous : *Token;
@@ -9,6 +9,15 @@ Parse_State :: struct {
ctx : *Compiler_Context;
}
AST_SENTINEL : AST_Node : .{
kind = .Invalid,
name = "__SENTINEL__",
};
is_sentinel :: inline (node : *AST_Node) -> bool {
return node == *AST_SENTINEL;
}
////////////////////////////
//@nb - Parsing helper types
Separator_Type :: enum {
@@ -39,7 +48,7 @@ Precedence :: enum {
PREC_PRIMARY;
}
Parse_Fn :: #type (parse_state: *Parse_State, left : *AST_Node) -> *AST_Node;
Parse_Fn :: #type (parse_state: *Parser, left : *AST_Node) -> *AST_Node;
Parse_Rule :: struct {
prefix : Parse_Fn;
infix : Parse_Fn;
@@ -96,7 +105,7 @@ parse_rules :: #run -> [(cast(int)Token_Kind.TOKEN_ERROR) + 1]Parse_Rule {
//@nb - Error handling functions
//nb - Record an error and report it immediately to the user.
record_error :: (parse_state : *Parse_State, token : Token, message : string, report_source_location : bool = true) {
record_error :: (parse_state : *Parser, token : Token, message : string, report_source_location : bool = true) {
error : Compiler_Message;
error.message_kind = .Error;
error.message = message;
@@ -121,7 +130,7 @@ record_error :: (parse_state : *Parse_State, token : Token, message : string, re
rewind_to_snapshot(parse_state, snap);
}
generate_source_location_from_token :: (state : *Parse_State, token : Token) -> Source_Range {
generate_source_location_from_token :: (state : *Parser, token : Token) -> Source_Range {
location : Source_Range;
begin : Token = token;
begin.index -= begin.column;
@@ -142,7 +151,7 @@ generate_source_location_from_token :: (state : *Parse_State, token : Token) ->
return location;
}
unexpected_token :: (state : *Parse_State, token : Token, message : string) {
unexpected_token :: (state : *Parser, token : Token, message : string) {
/*
*/
@@ -176,7 +185,7 @@ unexpected_token :: (state : *Parse_State, token : Token, message : string) {
record_error(state, token, final_message, false);
}
else_if_without_if :: (state : *Parse_State) {
else_if_without_if :: (state : *Parser) {
builder : String_Builder;
init_string_builder(*builder,, temp);
@@ -198,7 +207,7 @@ else_if_without_if :: (state : *Parse_State) {
record_error(state, token, final_message, false);
}
else_without_if :: (state : *Parse_State) {
else_without_if :: (state : *Parser) {
builder : String_Builder;
init_string_builder(*builder,, temp);
@@ -220,7 +229,7 @@ else_without_if :: (state : *Parse_State) {
record_error(state, token, final_message, false);
}
unable_to_parse_statement :: (state : *Parse_State, token : Token, message : string = "") {
unable_to_parse_statement :: (state : *Parser, token : Token, message : string = "") {
builder : String_Builder;
init_string_builder(*builder,, temp);
@@ -240,7 +249,7 @@ unable_to_parse_statement :: (state : *Parse_State, token : Token, message : str
record_error(state, token, final_message, false);
}
expected_expression :: (state : *Parse_State, token : Token, message : string) {
expected_expression :: (state : *Parser, token : Token, message : string) {
builder : String_Builder;
init_string_builder(*builder,, temp);
@@ -259,7 +268,7 @@ expected_expression :: (state : *Parse_State, token : Token, message : string) {
record_error(state, token, final_message, false);
}
missing_type_specifier :: (state : *Parse_State, token : Token, message : string) {
missing_type_specifier :: (state : *Parser, token : Token, message : string) {
builder : String_Builder;
init_string_builder(*builder,, temp);
@@ -283,7 +292,7 @@ missing_type_specifier :: (state : *Parse_State, token : Token, message : string
record_error(state, token, final_message, false);
}
empty_block :: (state : *Parse_State, token : Token, message : string) {
empty_block :: (state : *Parser, token : Token, message : string) {
builder : String_Builder;
init_string_builder(*builder,, temp);
@@ -307,7 +316,7 @@ empty_block :: (state : *Parse_State, token : Token, message : string) {
record_error(state, token, final_message, false);
}
unable_to_open_file :: (state : *Parse_State, path : string, token : Token) {
unable_to_open_file :: (state : *Parser, path : string, token : Token) {
builder : String_Builder;
init_string_builder(*builder,, temp);
@@ -327,7 +336,7 @@ unable_to_open_file :: (state : *Parse_State, path : string, token : Token) {
record_error(state, token, final_message, false);
}
entry_point_requires_return_value :: (state : *Parse_State, token : Token) {
entry_point_requires_return_value :: (state : *Parser, token : Token) {
builder : String_Builder;
init_string_builder(*builder,, temp);
@@ -346,7 +355,7 @@ entry_point_requires_return_value :: (state : *Parse_State, token : Token) {
record_error(state, token, final_message, false);
}
error_node :: (parse_state : *Parse_State, message : string) -> *AST_Node {
error_node :: (parse_state : *Parser, message : string) -> *AST_Node {
node := make_node(parse_state, .Error);
node.name = copy_string(message);
@@ -355,7 +364,7 @@ error_node :: (parse_state : *Parse_State, message : string) -> *AST_Node {
//nb - Advance to the next sync point.
// A sync point is the next token that ends a statement or starts/ends a block.
advance_to_sync_point :: (parse_state : *Parse_State) {
advance_to_sync_point :: (parse_state : *Parser) {
while true {
if parse_state.current.kind == .TOKEN_SEMICOLON || parse_state.current.kind == .TOKEN_RIGHTBRACE ||
parse_state.current.kind == .TOKEN_LEFTBRACE || parse_state.current.kind == .TOKEN_EOF {
@@ -379,7 +388,7 @@ make_node :: (nodes : *[..]AST_Node, kind : AST_Kind) -> *AST_Node {
return *(nodes.*[nodes.count - 1]);
}
make_node :: (parse_state : *Parse_State, kind : AST_Kind) -> *AST_Node {
make_node :: (parse_state : *Parser, kind : AST_Kind) -> *AST_Node {
return make_node(*parse_state.ctx.nodes, kind);
}
@@ -520,7 +529,7 @@ get_field_list :: (struct_or_func : *AST_Node) -> *AST_Node {
return struct_or_func.children[0];
}
add_child :: (node : *AST_Node, child : *AST_Node) {
add_child :: inline (node : *AST_Node, child : *AST_Node) {
child.parent = node;
array_add(*node.children, child);
}
@@ -531,7 +540,7 @@ Sync_Snapshot :: struct {
current_token_index : int;
}
snapshot_state :: (parse_state : *Parse_State) -> Sync_Snapshot {
snapshot_state :: (parse_state : *Parser) -> Sync_Snapshot {
snapshot : Sync_Snapshot;
snapshot.current = parse_state.current;
snapshot.previous = parse_state.previous;
@@ -540,13 +549,13 @@ snapshot_state :: (parse_state : *Parse_State) -> Sync_Snapshot {
return snapshot;
}
rewind_to_snapshot :: (parse_state : *Parse_State, snapshot : Sync_Snapshot) {
rewind_to_snapshot :: (parse_state : *Parser, snapshot : Sync_Snapshot) {
parse_state.current = snapshot.current;
parse_state.previous = snapshot.previous;
parse_state.current_token_index = snapshot.current_token_index;
}
advance :: (parse_state : *Parse_State) {
advance :: (parse_state : *Parser) {
parse_state.previous = parse_state.current;
while true {
@@ -564,18 +573,18 @@ advance :: (parse_state : *Parse_State) {
}
//nb - Checks if the current token is of a certain kind and advances if it is
match :: (parse_state : *Parse_State, kind : Token_Kind) -> bool {
match :: (parse_state : *Parser, kind : Token_Kind) -> bool {
if !check(parse_state, kind) return false;
advance(parse_state);
return true;
}
//nb - Checks if the current token is of a certain kind
check :: (parse_state : *Parse_State, kind : Token_Kind) -> bool {
check :: (parse_state : *Parser, kind : Token_Kind) -> bool {
return parse_state.current.kind == kind;
}
check_any :: (parse_state : *Parse_State, kinds : ..Token_Kind) -> bool {
check_any :: (parse_state : *Parser, kinds : ..Token_Kind) -> bool {
for kind : kinds {
if check(parse_state, kind) {
return true;
@@ -585,12 +594,12 @@ check_any :: (parse_state : *Parse_State, kinds : ..Token_Kind) -> bool {
}
//nb - Checks if the next token is of a certain kind
check_next :: (parse_state : *Parse_State, kind : Token_Kind) -> bool {
check_next :: (parse_state : *Parser, kind : Token_Kind) -> bool {
return parse_state.ctx.tokens[parse_state.current_token_index].kind == kind;
}
//nb - Consume a token if
consume :: (parse_state : *Parse_State, kind : Token_Kind, message : string) {
consume :: (parse_state : *Parser, kind : Token_Kind, message : string) {
if parse_state.current.kind == kind {
advance(parse_state);
return;
@@ -612,7 +621,7 @@ get_rule :: (kind : Token_Kind) -> *Parse_Rule {
return *parse_rules[kind];
}
precedence :: (parse_state : *Parse_State, precedence : Precedence, message : string = "") -> *AST_Node {
precedence :: (parse_state : *Parser, precedence : Precedence, message : string = "") -> *AST_Node {
prev := parse_state.previous;
advance(parse_state);
@@ -640,7 +649,7 @@ precedence :: (parse_state : *Parse_State, precedence : Precedence, message : st
tok_s.count = parse_state.previous.length;
expected_expression(parse_state, parse_state.current, tprint("Reached end of file. Expected expression after '%'.", tok_s));
// @Incomplete: Add error node here?
return null;
return *AST_SENTINEL;
}
infix_rule := get_rule(parse_state.previous.kind).infix;
left = infix_rule(parse_state, left);
@@ -649,7 +658,7 @@ precedence :: (parse_state : *Parse_State, precedence : Precedence, message : st
return left;
}
named_variable :: (parse_state : *Parse_State, left : *AST_Node) -> *AST_Node {
named_variable :: (parse_state : *Parser, left : *AST_Node) -> *AST_Node {
if check(parse_state, .TOKEN_LEFTPAREN) {
return call(parse_state, left);
}
@@ -662,7 +671,7 @@ named_variable :: (parse_state : *Parse_State, left : *AST_Node) -> *AST_Node {
return variable;
}
binary :: (parse_state : *Parse_State, left : *AST_Node) -> *AST_Node {
binary :: (parse_state : *Parser, left : *AST_Node) -> *AST_Node {
op := parse_state.previous.*;
rule := get_rule(op.kind);
@@ -704,7 +713,7 @@ binary :: (parse_state : *Parse_State, left : *AST_Node) -> *AST_Node {
return binary_expression;
}
array_access :: (parse_state : *Parse_State, left : *AST_Node) -> *AST_Node {
array_access :: (parse_state : *Parser, left : *AST_Node) -> *AST_Node {
identifier := parse_state.ctx.tokens[parse_state.current_token_index - 3];
left_bracket := parse_state.ctx.tokens[parse_state.current_token_index - 2];
@@ -734,7 +743,7 @@ array_access :: (parse_state : *Parse_State, left : *AST_Node) -> *AST_Node {
return array_access;
}
unary :: (parse_state : *Parse_State, left : *AST_Node) -> *AST_Node {
unary :: (parse_state : *Parser, left : *AST_Node) -> *AST_Node {
op := parse_state.previous.*;
rule := get_rule(op.kind);
@@ -755,13 +764,13 @@ unary :: (parse_state : *Parse_State, left : *AST_Node) -> *AST_Node {
return unary_expression;
}
grouping :: (parse_state : *Parse_State, left : *AST_Node) -> *AST_Node {
grouping :: (parse_state : *Parser, left : *AST_Node) -> *AST_Node {
grouping := expression(parse_state);
consume(parse_state, .TOKEN_RIGHTPAREN, "Expect ')' after group expression.");
return grouping;
}
directive :: (state : *Parse_State) -> *AST_Node {
directive :: (state : *Parser) -> *AST_Node {
if state.current.ident_value == "foreign" {
advance(state);
identifier_token := state.current;
@@ -801,15 +810,12 @@ directive :: (state : *Parse_State) -> *AST_Node {
return if_directive;
}
return null;
return *AST_SENTINEL;
}
call :: (parse_state : *Parse_State, left : *AST_Node) -> *AST_Node {
call :: (parse_state : *Parser, left : *AST_Node) -> *AST_Node {
call := make_node(parse_state, .Call);
source_location := generate_source_location_from_token(parse_state, parse_state.previous);
// source_location : Source_Range;
// source_location.begin = parse_state.previous;
// source_location.main_token = parse_state.previous;
prev := parse_state.previous;
call.name = prev.ident_value;
@@ -830,7 +836,7 @@ call :: (parse_state : *Parse_State, left : *AST_Node) -> *AST_Node {
return call;
}
dot :: (parse_state : *Parse_State, left : *AST_Node) -> *AST_Node {
dot :: (parse_state : *Parser, left : *AST_Node) -> *AST_Node {
consume(parse_state, .TOKEN_IDENTIFIER, "Expect property name after '.'.");
identifier := parse_state.previous;
@@ -863,7 +869,7 @@ dot :: (parse_state : *Parse_State, left : *AST_Node) -> *AST_Node {
return access;
}
integer :: (parse_state : *Parse_State, left : *AST_Node) -> *AST_Node {
integer :: (parse_state : *Parser, left : *AST_Node) -> *AST_Node {
value := parse_state.previous.integer_value;
node := make_node(parse_state, .Integer);
node.source_location.begin = parse_state.previous;
@@ -873,7 +879,7 @@ integer :: (parse_state : *Parse_State, left : *AST_Node) -> *AST_Node {
return node;
}
floating :: (parse_state : *Parse_State, left : *AST_Node) -> *AST_Node {
floating :: (parse_state : *Parser, left : *AST_Node) -> *AST_Node {
value := parse_state.previous.float_value;
node := make_node(parse_state, .Float);
node.source_location.begin = parse_state.previous;
@@ -883,7 +889,7 @@ floating :: (parse_state : *Parse_State, left : *AST_Node) -> *AST_Node {
return node;
}
expression :: (parse_state : *Parse_State, message : string = "") -> *AST_Node {
expression :: (parse_state : *Parser, message : string = "") -> *AST_Node {
expression := precedence(parse_state, .PREC_ASSIGNMENT, message);
return expression;
}
@@ -891,7 +897,7 @@ expression :: (parse_state : *Parse_State, message : string = "") -> *AST_Node {
////////////////////////////
//@nb - Statement parsing functions
field_assignment :: (parse_state : *Parse_State, identifier_token : *Token) -> *AST_Node {
field_assignment :: (parse_state : *Parser, identifier_token : *Token) -> *AST_Node {
node : *AST_Node = make_node(parse_state, .Field);
identifier := identifier_token.*;
@@ -910,7 +916,7 @@ field_assignment :: (parse_state : *Parse_State, identifier_token : *Token) -> *
}
//nb - Non-const field declarations
field_declaration :: (parse_state : *Parse_State, identifier_token : *Token) -> *AST_Node {
field_declaration :: (parse_state : *Parser, identifier_token : *Token) -> *AST_Node {
node : *AST_Node = make_node(parse_state, .Field);
identifier := identifier_token.*;
@@ -971,7 +977,7 @@ field_declaration :: (parse_state : *Parse_State, identifier_token : *Token) ->
return node;
}
argument_list :: (parse_state : *Parse_State) -> *AST_Node {
argument_list :: (parse_state : *Parser) -> *AST_Node {
node : *AST_Node;
source_location : Source_Range;
@@ -1019,7 +1025,7 @@ argument_list :: (parse_state : *Parse_State) -> *AST_Node {
return node;
}
expression_statement :: (parse_state : *Parse_State) -> *AST_Node {
expression_statement :: (parse_state : *Parser) -> *AST_Node {
node := make_node(parse_state, .Expression_Statement);
source_location : Source_Range;
@@ -1037,7 +1043,7 @@ expression_statement :: (parse_state : *Parse_State) -> *AST_Node {
return node;
}
statement :: (parse_state : *Parse_State) -> *AST_Node {
statement :: (parse_state : *Parser) -> *AST_Node {
if match(parse_state, .TOKEN_RETURN) {
node := make_node(parse_state, .Return);
@@ -1155,7 +1161,7 @@ statement :: (parse_state : *Parse_State) -> *AST_Node {
return error_node(parse_state, "Couldn't parse statement.");
}
else_statement :: (parse_state : *Parse_State) -> *AST_Node {
else_statement :: (parse_state : *Parser) -> *AST_Node {
if check(parse_state, .TOKEN_IF) {
return statement(parse_state);
} else if check(parse_state, .TOKEN_DIRECTIVE) && parse_state.current.ident_value == "if" {
@@ -1164,7 +1170,7 @@ else_statement :: (parse_state : *Parse_State) -> *AST_Node {
return block(parse_state);
}
block :: (parse_state : *Parse_State) -> *AST_Node {
block :: (parse_state : *Parser) -> *AST_Node {
node : *AST_Node = make_node(parse_state, .Block);
array_reserve(*node.children, 32);
@@ -1186,7 +1192,7 @@ block :: (parse_state : *Parse_State) -> *AST_Node {
return node;
}
field_list :: (parse_state : *Parse_State, separator : Separator_Type, require_field_names := true) -> *AST_Node {
field_list :: (parse_state : *Parser, separator : Separator_Type, require_field_names := true) -> *AST_Node {
node : *AST_Node = make_node(parse_state, .FieldList);
array_reserve(*node.children, 16);
source_location : Source_Range;
@@ -1231,7 +1237,7 @@ field_list :: (parse_state : *Parse_State, separator : Separator_Type, require_f
return node;
}
function_declaration :: (parse_state : *Parse_State, identifier_token : *Token, entry_point_kind : Entry_Point_Type, expect_body : bool = true, require_field_names : bool = true) -> *AST_Node {
function_declaration :: (parse_state : *Parser, identifier_token : *Token, entry_point_kind : Entry_Point_Type, expect_body : bool = true, require_field_names : bool = true) -> *AST_Node {
node : *AST_Node;
source_location : Source_Range;
@@ -1302,7 +1308,7 @@ function_declaration :: (parse_state : *Parse_State, identifier_token : *Token,
return node;
}
buffer :: (state : *Parse_State, identifier_token : *Token = null) -> *AST_Node {
buffer :: (state : *Parser, identifier_token : *Token = null) -> *AST_Node {
node : *AST_Node = make_node(state, .Buffer);
source_location : Source_Range;
source_location.begin = state.current;
@@ -1334,7 +1340,7 @@ buffer :: (state : *Parse_State, identifier_token : *Token = null) -> *AST_Node
return node;
}
constant_buffer :: (parse_state : *Parse_State, identifier_token : *Token = null) -> *AST_Node {
constant_buffer :: (parse_state : *Parser, identifier_token : *Token = null) -> *AST_Node {
node : *AST_Node = make_node(parse_state, .CBuffer);
source_location : Source_Range;
source_location.begin = parse_state.current;
@@ -1365,7 +1371,7 @@ constant_buffer :: (parse_state : *Parse_State, identifier_token : *Token = null
return node;
}
struct_declaration :: (parse_state : *Parse_State, identifier_token : *Token) -> *AST_Node {
struct_declaration :: (parse_state : *Parser, identifier_token : *Token) -> *AST_Node {
source_location := generate_source_location_from_token(parse_state, identifier_token);
consume(parse_state, .TOKEN_LEFTBRACE, "Expect '{' before struct declaration.");
@@ -1383,13 +1389,13 @@ struct_declaration :: (parse_state : *Parse_State, identifier_token : *Token) ->
return node;
}
access :: (parse_state : *Parse_State, identifier_token : *Token) -> *AST_Node {
access :: (parse_state : *Parser, identifier_token : *Token) -> *AST_Node {
err_node := error_node(parse_state, tprint("Accessors not yet implemented. Token: %.", identifier_token.ident_value));
advance(parse_state);
return err_node;
}
const_declaration :: (parse_state : *Parse_State, identifier_token : *Token) -> *AST_Node {
const_declaration :: (parse_state : *Parser, identifier_token : *Token) -> *AST_Node {
if match(parse_state, .TOKEN_STRUCT) {
return struct_declaration(parse_state, identifier_token);
} else if check(parse_state, .TOKEN_LEFTPAREN) {
@@ -1402,7 +1408,7 @@ const_declaration :: (parse_state : *Parse_State, identifier_token : *Token) ->
return error_node(parse_state, tprint("Couldn't parse constant declaration at token %\n", parse_state.current.*));
}
declaration :: (parse_state : *Parse_State) -> *AST_Node {
declaration :: (parse_state : *Parser) -> *AST_Node {
skip_statement := false;
decl_node : *AST_Node;
if match(parse_state, .TOKEN_VERTEX) {
@@ -1468,7 +1474,7 @@ parse :: (ctx : *Compiler_Context, allocator := temp) {
init_context_allocators();
defer clear_context_allocators();
parse_state : Parse_State;
parse_state : Parser;
array_reserve(*ctx.nodes, 4096);
parse_state.current_token_index = 0;
parse_state.ctx = ctx;