diff --git a/depend b/depend index 96d0485d52..8db2d7a675 100644 --- a/depend +++ b/depend @@ -16094,6 +16094,7 @@ scheduler.$(OBJEXT): $(top_srcdir)/internal/sanitizers.h scheduler.$(OBJEXT): $(top_srcdir)/internal/serial.h scheduler.$(OBJEXT): $(top_srcdir)/internal/set_table.h scheduler.$(OBJEXT): $(top_srcdir)/internal/static_assert.h +scheduler.$(OBJEXT): $(top_srcdir)/internal/struct.h scheduler.$(OBJEXT): $(top_srcdir)/internal/thread.h scheduler.$(OBJEXT): $(top_srcdir)/internal/variable.h scheduler.$(OBJEXT): $(top_srcdir)/internal/vm.h @@ -16897,6 +16898,7 @@ signal.$(OBJEXT): $(top_srcdir)/internal/set_table.h signal.$(OBJEXT): $(top_srcdir)/internal/signal.h signal.$(OBJEXT): $(top_srcdir)/internal/static_assert.h signal.$(OBJEXT): $(top_srcdir)/internal/string.h +signal.$(OBJEXT): $(top_srcdir)/internal/struct.h signal.$(OBJEXT): $(top_srcdir)/internal/thread.h signal.$(OBJEXT): $(top_srcdir)/internal/variable.h signal.$(OBJEXT): $(top_srcdir)/internal/vm.h diff --git a/gc.c b/gc.c index e521369caa..b29c72d7df 100644 --- a/gc.c +++ b/gc.c @@ -1106,6 +1106,16 @@ VALUE class_allocate_complex_instance(VALUE klass, uint32_t capacity) return obj; } +static inline size_t +robject_embedded_size(uint32_t fields_count) +{ + size_t size = rb_obj_embedded_size(fields_count); + if (!rb_gc_size_allocatable_p(size)) { + size = sizeof(struct RObject); + } + return size; +} + VALUE rb_class_allocate_instance(VALUE klass) { @@ -1118,10 +1128,7 @@ rb_class_allocate_instance(VALUE klass) obj = class_allocate_complex_instance(klass, index_tbl_num_entries); } else { - size_t size = rb_obj_embedded_size(index_tbl_num_entries); - if (!rb_gc_size_allocatable_p(size)) { - size = sizeof(struct RObject); - } + size_t size = robject_embedded_size(index_tbl_num_entries); // There might be a NEWOBJ tracepoint callback, and it may set fields. // So the shape must be passed to `NEWOBJ_OF`. @@ -1145,6 +1152,25 @@ rb_class_allocate_instance(VALUE klass) return obj; } +#if USE_ZJIT +bool +rb_zjit_class_allocate_instance_fastpath(VALUE klass, size_t *size_out, shape_id_t *shape_id_out) +{ + uint32_t index_tbl_num_entries = RCLASS_MAX_IV_COUNT(klass); + + RUBY_ASSERT(rb_shape_max_capacity() > 0); + if (RB_UNLIKELY(index_tbl_num_entries > rb_shape_max_capacity())) { + return false; + } + + size_t size = robject_embedded_size(index_tbl_num_entries); + *size_out = size; + *shape_id_out = rb_shape_transition_slot_size(rb_shape_transition_robject(0), + rb_gc_size_slot_size(size)); + return true; +} +#endif + void rb_gc_register_pinning_obj(VALUE obj) { diff --git a/zjit.h b/zjit.h index 586cf28a20..2bbac39b7c 100644 --- a/zjit.h +++ b/zjit.h @@ -4,6 +4,8 @@ // This file contains definitions ZJIT exposes to the CRuby codebase // +#include "shape.h" // for shape_id_t + // ZJIT_STATS controls whether to support runtime counters in the interpreter #ifndef ZJIT_STATS # define ZJIT_STATS (USE_ZJIT && RUBY_DEBUG) @@ -91,6 +93,7 @@ void rb_zjit_invalidate_root_box(void); void rb_zjit_jit_frame_update_references(zjit_jit_frame_t *jit_frame); void rb_zjit_materialize_frames(const rb_execution_context_t *ec, rb_control_frame_t *cfp); size_t rb_zjit_hash_new_size(void); +bool rb_zjit_class_allocate_instance_fastpath(VALUE klass, size_t *size_out, shape_id_t *shape_id_out); // Special value for cfp->jit_return that means "this is a C method frame, use // rb_zjit_c_frame as the JITFrame". We don't control the native stack layout diff --git a/zjit/bindgen/src/main.rs b/zjit/bindgen/src/main.rs index cc28a6e05e..943ae22e30 100644 --- a/zjit/bindgen/src/main.rs +++ b/zjit/bindgen/src/main.rs @@ -112,6 +112,7 @@ fn main() { .allowlist_function("rb_zjit_profile_disable") .allowlist_function("rb_zjit_insn_to_bare_insn") .allowlist_function("rb_zjit_hash_new_size") + .allowlist_function("rb_zjit_class_allocate_instance_fastpath") // For crashing .allowlist_function("rb_bug") diff --git a/zjit/src/codegen.rs b/zjit/src/codegen.rs index 7f62d83ae0..25f3d77be2 100644 --- a/zjit/src/codegen.rs +++ b/zjit/src/codegen.rs @@ -627,7 +627,7 @@ fn gen_insn(cb: &mut CodeBlock, jit: &mut JITState, asm: &mut Assembler, functio Insn::ArrayPop { array, state } => gen_array_pop(asm, opnd!(array), &function.frame_state(*state)), Insn::ArrayLength { array } => gen_array_length(asm, opnd!(array)), Insn::ObjectAlloc { val, state } => gen_object_alloc(jit, asm, function, opnd!(val), &function.frame_state(*state)), - &Insn::ObjectAllocClass { class, state } => gen_object_alloc_class(asm, class, &function.frame_state(state)), + &Insn::ObjectAllocClass { class, state } => gen_object_alloc_class(jit, asm, class, &function.frame_state(state)), Insn::StringCopy { val, chilled, state } => gen_string_copy(asm, opnd!(val), *chilled, &function.frame_state(*state)), Insn::StringConcat { strings, state } => gen_string_concat(jit, asm, function, opnds!(strings), &function.frame_state(*state)), &Insn::StringGetbyte { string, index } => gen_string_getbyte(asm, opnd!(string), opnd!(index)), @@ -2344,13 +2344,24 @@ fn gen_object_alloc(jit: &JITState, asm: &mut Assembler, function: &Function, va asm_ccall!(asm, rb_obj_alloc, val) } -fn gen_object_alloc_class(asm: &mut Assembler, class: VALUE, state: &FrameState) -> lir::Opnd { +fn gen_object_alloc_class(jit: &mut JITState, asm: &mut Assembler, class: VALUE, state: &FrameState) -> lir::Opnd { // Allocating an object for a known class with default allocator is leaf; see doc for // `ObjectAllocClass`. gen_prepare_leaf_call_with_gc(asm, state); if unsafe { rb_zjit_class_has_default_allocator(class) } { - // TODO(max): inline code to allocate an instance - asm_ccall!(asm, rb_class_allocate_instance, class.into()) + let mut alloc_size: usize = 0; + let mut shape_id: shape_id_t = 0; + let has_fastpath = unsafe { + rb_zjit_class_allocate_instance_fastpath(class, &mut alloc_size, &mut shape_id) + }; + if has_fastpath { + let flags = (RUBY_T_OBJECT as u64) | ((shape_id as u64) << RB_SHAPE_FLAG_SHIFT as u64); + gc_fastpath::gc_fast_path_new_obj(jit, asm, alloc_size, flags, class, |asm| { + asm_ccall!(asm, rb_class_allocate_instance, class.into()) + }) + } else { + asm_ccall!(asm, rb_class_allocate_instance, class.into()) + } } else { assert!(class_has_leaf_allocator(class), "class passed to ObjectAllocClass must have a leaf allocator"); let alloc_func = unsafe { rb_zjit_class_get_alloc_func(class) }; diff --git a/zjit/src/codegen_tests.rs b/zjit/src/codegen_tests.rs index 51055badf2..ff4cf77891 100644 --- a/zjit/src/codegen_tests.rs +++ b/zjit/src/codegen_tests.rs @@ -2920,6 +2920,32 @@ fn test_new_hash_empty_gc_stress() { "#), @"[Hash, 1, nil, {a: 1}]"); } +#[test] +fn test_object_alloc_gc_stress() { + eval(" + class Foo + def initialize + @a = 1 + @b = 2 + end + def sum = @a + @b + end + def make = Foo.new + "); + assert_contains_opcode("make", YARVINSN_opt_new); + assert_snapshot!(assert_compiles(r#" + begin + GC.stress = true + make + foo = make + foo.instance_variable_set(:@c, 3) + [foo.class, foo.sum, foo.instance_variables] + ensure + GC.stress = false + end + "#), @"[Foo, 3, [:@a, :@b, :@c]]"); +} + #[test] fn test_new_hash_nonempty() { eval(r#" diff --git a/zjit/src/cruby_bindings.inc.rs b/zjit/src/cruby_bindings.inc.rs index 7b000daa73..e54937dd94 100644 --- a/zjit/src/cruby_bindings.inc.rs +++ b/zjit/src/cruby_bindings.inc.rs @@ -2234,6 +2234,11 @@ unsafe extern "C" { pub fn rb_iseq_label(iseq: *const rb_iseq_t) -> VALUE; pub fn rb_iseq_defined_string(type_: defined_type) -> VALUE; pub fn rb_zjit_hash_new_size() -> usize; + pub fn rb_zjit_class_allocate_instance_fastpath( + klass: VALUE, + size_out: *mut usize, + shape_id_out: *mut shape_id_t, + ) -> bool; pub fn rb_profile_frames( start: ::std::os::raw::c_int, limit: ::std::os::raw::c_int,