Much better allocation strategy with new ncore arenas. Add more builtin node gen stuff. It's kind of wonky.

Ink.jai

@@ -159,7 +159,7 @@ do_golden_comparison :: (golden_path : string, comparison_text : string, result_
 
 run_codegen_test :: (file_path : string, result : *Compile_Result, output_type : Output_Type = 0) -> Result {
 	result.file = make_file(result, file_path);
-	result.allocator = make_arena(*result.arena);
+	result.allocator = make_arena(Megabytes(128));
 
 	result_data : Result;
 	result_data.path = file_path;
@@ -225,7 +225,7 @@ run_lexer_test :: (file_path : string, result : *Compile_Result, output_type : O
 	result_text : string;
 
 	result.file = make_file(result, file_path);
-	result.allocator = make_arena(*result.arena);
+	result.allocator = make_arena(Megabytes(128));
 	
 	lex(result);
 	if result.had_error {
@@ -251,7 +251,7 @@ run_parser_test :: (file_path : string, result : *Compile_Result, output_type :
 	result_data.path = file_path;
 
 	result.file = make_file(result, file_path);
-	result.allocator = make_arena(*result.arena);
+	result.allocator = make_arena(Megabytes(128));
 
 	lex(result);
 	if result.had_error {
@@ -315,7 +315,7 @@ run_semantic_analysis_test :: (result : *Compile_Result, output_type : Output_Ty
 
 run_semantic_analysis_test :: (file_path : string, result : *Compile_Result, output_type : Output_Type = 0) -> Result {
 	result.file = make_file(result, file_path);
-	result.allocator = make_arena(*result.arena);
+	result.allocator = make_arena(Megabytes(128));
 
 	result_data : Result;
 	result_data.path = file_path;
@@ -345,7 +345,7 @@ run_test_new :: (file_path : string, stage_flags : Stage_Flags, results : *[..]R
 	compile_result : Compile_Result;
 	
 	compile_result.file = make_file(*compile_result, file_path);
-	compile_result.allocator = make_arena(*compile_result.arena);
+	compile_result.allocator = make_arena(Megabytes(128));
 	
 	result : Result;
 	if stage_flags & .Lexer {

Lexing.jai

@@ -572,6 +572,8 @@ lex :: (result : *Compile_Result) {
 
 	lexer : Lexer;
 	lexer.result = result;
+	lexer.result.tokens.allocator = result.allocator;
+	array_reserve(*lexer.result.tokens, 1024 * 1024);
 	
 	init_lexer_from_string(*lexer, result.file.source);
 	lexer.path = result.file.path;

Parsing.jai

@@ -388,9 +388,84 @@ make_node :: (parse_state : *Parse_State, kind : AST_Kind) -> *AST_Node {
 // 	return node;
 // }
 
-// new_builtin_struct_node :: (nodes : *[..]AST_Node, name : string, member_names : []string, allocator : Allocator) -> *AST_Node {
+make_builtin_token :: (builder : *String_Builder, kind : Token_Kind, text : string, col : *int, line : *int) -> Token {
+	tok : Token;
+	tok.kind = kind;
 
-// }
+	start := 0;
+
+	buffer := get_current_buffer(builder);
+
+	if buffer {
+		start := buffer.count;
+	} 
+	
+	print_to_builder(builder, "%", text);
+	buffer = get_current_buffer(builder);
+	end := buffer.count;
+
+	for c : text {
+		if c == #char "\n" {
+			line.* ++ 1;
+			col.* = 0;
+		} else {
+			col.* += 1;
+		}
+	}
+
+	tok.column = col.*;	
+	tok.index  = buffer.count;
+	tok.length = text.count;
+	tok.builtin = true;
+
+	return tok;
+}
+
+new_builtin_struct_node :: (result : *Compile_Result, name : string, members : []Arg, allocator : Allocator) -> *AST_Node {
+	sc := get_scratch(result, allocator);
+	defer scratch_end(sc);
+	builder : String_Builder;
+	builder.allocator = sc.allocator; // I want to find a good way to use scratch here...
+
+	node := make_node(*result.nodes, .Struct, allocator);
+
+	col   := 0;
+	line  := 0;
+
+	tok_index := result.tokens.count;
+
+	array_add(*result.tokens, make_builtin_token(*builder, .TOKEN_IDENTIFIER, tprint("%", name), *col, *line));
+	append(*builder, " ");
+	array_add(*result.tokens, make_builtin_token(*builder, .TOKEN_DOUBLECOLON, "::", *col, *line));
+	append(*builder, " ");
+	array_add(*result.tokens, make_builtin_token(*builder, .TOKEN_STRUCT, "struct", *col, *line));
+	append(*builder, " ");
+	array_add(*result.tokens, make_builtin_token(*builder, .TOKEN_LEFTBRACE, "{", *col, *line));
+	append(*builder, "\n");
+
+	for member : members {
+		// @Incomplete: Missing field list
+		array_add(*result.tokens, make_builtin_token(*builder, .TOKEN_IDENTIFIER, tprint("%", member.name), *col, *line));
+		append(*builder, " ");
+		array_add(*result.tokens, make_builtin_token(*builder, .TOKEN_COLON, ":", *col, *line));
+		append(*builder, " ");
+		array_add(*result.tokens, make_builtin_token(*builder, .TOKEN_IDENTIFIER, tprint("%", member.typename), *col, *line));
+		array_add(*result.tokens, make_builtin_token(*builder, .TOKEN_SEMICOLON, ";", *col, *line));
+		append(*builder, "\n");
+	}
+
+	array_add(*result.tokens, make_builtin_token(*builder, .TOKEN_RIGHTBRACE, "}", *col, *line));
+	append(*builder, "\n");
+
+	source := builder_to_string(*builder,, allocator);
+
+	for i : tok_index..result.tokens.count - 1 {
+		tok := *result.tokens[i];
+		tok.source = *source.data[tok.column];
+	}
+
+	return node;
+}
 
 add_child :: (node : *AST_Node, child : *AST_Node) {
 	child.parent = node;

Semantic_Analysis.jai

@@ -149,9 +149,6 @@ Semantic_Checker :: struct {
 	current_sampler_index  : u32 = 0;
 	current_texture_index  : u32 = 0;
 
-	messages          : [..]Compiler_Message;
-	message_arena     : Arena;
-	message_allocator : Allocator;
 	had_error         : bool;
 }
 
@@ -595,7 +592,7 @@ record_error :: (checker : *Semantic_Checker, error_string : string, locations :
 	error.message = error_string;
 	
 	checker.had_error = true;
- 	array_add(*checker.messages, error);
+ 	array_add(*checker.result.messages, error);
 }
 
 is_proper :: (var : Type_Variable) -> bool {
@@ -727,7 +724,7 @@ new_builtin_struct :: (checker : *Semantic_Checker, name : string, members : []A
 	// @Incomplete: Skip for now. This is solely for error reporting?
 	//              At least let's not make a big deal out of it for now.
 	//              We could report builtin nodes in a special way instead of with an actual source location.
-	// builtin_node := new_builtin_struct_node(checker.result.nodes, members, checker.result.allocator;
+	builtin_node := new_builtin_struct_node(checker.result, name, members, checker.result.allocator);
 	
 	symbol : Defined_Symbol;
 	symbol.name = name;
@@ -808,15 +805,14 @@ init_semantic_checker :: (checker : *Semantic_Checker, root : *AST_Node, path :
 	checker.current_sampler_index = 0;
 	checker.current_texture_index = 0;
 	
-	array_reserve(*checker.messages, 16);
-	
 	checker.program_root = root;
 	checker.path         = path;
 
 	// @Incomplete(niels): Use other allocator and/or add static array with convenience functions
+	checker.result.type_variables.allocator = checker.result.allocator;
 	array_reserve(*checker.result.type_variables, 2048);
 
-	checker.result.scope_stack.allocator = make_arena(*checker.result.scope_stack.arena);
+	checker.result.scope_stack.allocator = make_arena(Megabytes(8));
 	array_reserve(*checker.result.scope_stack.stack, 256);
 
 	global_scope, global_handle := push_scope(checker, kind = .Global);
@@ -1780,9 +1776,9 @@ add_builtins_new :: (checker : *Semantic_Checker) {
 
 	checker.state = .Adding_Builtins;
 	float_name := Typenames[Type_Kind.Float];
-	// new_builtin_struct(checker, "float2", .[.{"x", float_name}, .{"y", float_name}]);
+	new_builtin_struct(checker, "float2", .[.{"x", float_name}, .{"y", float_name}]);
-	// new_builtin_struct(checker, "float3", .[.{"x", float_name}, .{"y", float_name}, .{"z", float_name}]);
+	new_builtin_struct(checker, "float3", .[.{"x", float_name}, .{"y", float_name}, .{"z", float_name}]);
-	// new_builtin_struct(checker, "float4", .[.{"x", float_name}, .{"y", float_name}, .{"z", float_name}, .{"w", float_name}]);
+	new_builtin_struct(checker, "float4", .[.{"x", float_name}, .{"y", float_name}, .{"z", float_name}, .{"w", float_name}]);
 
 	checker.state = .Type_Checking;
 }
@@ -1814,9 +1810,22 @@ add_builtins :: (checker : *Semantic_Checker)  {
 	checker.state = .Adding_Builtins;
 
 	checker.result.file = make_file_from_string(BUILTIN);
-	checker.result.allocator = make_arena(*checker.result.arena);
+	checker.result.allocator = make_arena(Megabytes(128));
 
-	lexer : Lexer;
+	prev_file := checker.result.file;
+	prev_root := checker.result.root;
+	prev_tokens := checker.result.tokens;
+
+	checker.result.root = null;
+
+	tokens : [..]Token;
+	scratch := get_scratch(checker.result);
+	defer scratch_end(scratch);
+	tokens.allocator = scratch.allocator;
+	array_reserve(*tokens, 1024 * 1024);
+	checker.result.tokens = tokens;
+	
+	checker.result.tokens.count = 0;
 
 	lex(checker.result);
 	parse(checker.result);
@@ -1827,6 +1836,10 @@ add_builtins :: (checker : *Semantic_Checker)  {
 	}
 
 	checker.state = .Type_Checking;
+
+	checker.result.file   = prev_file;
+	checker.result.root   = prev_root;
+	checker.result.tokens = prev_tokens;
 }
 
 type_check :: (checker : *Semantic_Checker, root : *AST_Node) {
@@ -1845,23 +1858,14 @@ check :: (result : *Compile_Result) {
 	checker.current_texture_index = 0;
 	checker.result = result;
 
-	file := result.file;
-	root := result.root;
-	array_reserve(*checker.messages, 32);
-
-	result.root = null;
-	add_builtins_new(*checker);
-	add_builtins(*checker);
-
 	init_semantic_checker(*checker, result.root, result.file.path);
 	
-	checker.result.file = file;
+	// add_builtins_new(*checker);
-	result.root = root;
+	add_builtins(*checker);
 
 	type_check(*checker, result.root);
 	
 	result.had_error |= checker.had_error;
-	copy_messages(checker.messages, *result.messages);
 }
 
 // ===========================================================

module.jai

@@ -181,7 +181,23 @@ Compile_Result :: struct {
 	messages : [..]Compiler_Message;
 
 	allocator : Allocator;
-	arena     : Arena;
+	// arena     : Arena;
+
+	// scratch_arenas     : [2]Arena;
+	scratch_allocators : [2]Allocator;
+}
+
+get_scratch :: (result : *Compile_Result, conflict : Allocator = .{}) -> Scratch {
+	arena := cast(*Arena)conflict.data;
+	if result.scratch_allocators[0].data == null {
+		result.scratch_allocators[0] = make_arena(Megabytes(128));
+		result.scratch_allocators[1] = make_arena(Megabytes(128));
+	}
+	
+	if arena == get_arena(result.scratch_allocators[0]) {
+		return scratch_begin(*result.scratch_allocators[1]);
+	}
+	return scratch_begin(*result.scratch_allocators[0]);
 }
 
 record_error :: (result : *Compile_Result, format : string, args : .. Any) {
@@ -481,7 +497,7 @@ generate_output_data :: (result : *Compile_Result) {
 compile_file :: (compiler : *Shader_Compiler, path : string) -> Compile_Result {
 	result : Compile_Result;
 
-	result.allocator = make_arena(*result.arena);
+	result.allocator = make_arena(Megabytes(128));
 
 	result.file = make_file(*result, path);
 	result.environment = compiler.environment;