Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Use atomics for call cache #14338

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 5 commits into
base: master
Choose a base branch
from
Open

Use atomics for call cache #14338

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
8fb5016
Use atomics for inline call caches
jhawthorn Aug 20, 2025
115bae9
Use atomics for gccct
jhawthorn Aug 21, 2025
c376684
Only set cached flag when unset
jhawthorn Aug 21, 2025
331cb17
Update TSan suppressions
jhawthorn Aug 26, 2025
e903b47
Test?
jhawthorn Aug 26, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
Use atomics for inline call caches
  • Loading branch information
@jhawthorn
jhawthorn committed Aug 26, 2025
commit 8fb5016e495c44f5120738a68c114799dbb2f61c
8 changes: 4 additions & 4 deletions insns.def
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -874,8 +874,8 @@ sendforward
val = vm_sendish(ec, GET_CFP(), &adjusted_cd.cd, bh, mexp_search_method);
JIT_EXEC(ec, val);

if (cd->cc != adjusted_cd.cd.cc && vm_cc_markable(adjusted_cd.cd.cc)) {
RB_OBJ_WRITE(GET_ISEQ(), &cd->cc, adjusted_cd.cd.cc);
if (vm_cd_cc_load(cd) != adjusted_cd.cd.cc && vm_cc_markable(adjusted_cd.cd.cc)) {
vm_cd_cc_store((VALUE)GET_ISEQ(), cd, adjusted_cd.cd.cc);
}

if (UNDEF_P(val)) {
Expand Down Expand Up @@ -1114,8 +1114,8 @@ invokesuperforward
val = vm_sendish(ec, GET_CFP(), &adjusted_cd.cd, bh, mexp_search_super);
JIT_EXEC(ec, val);

if (cd->cc != adjusted_cd.cd.cc && vm_cc_markable(adjusted_cd.cd.cc)) {
RB_OBJ_WRITE(GET_ISEQ(), &cd->cc, adjusted_cd.cd.cc);
if (vm_cd_cc_load(cd) != adjusted_cd.cd.cc && vm_cc_markable(adjusted_cd.cd.cc)) {
vm_cd_cc_store((VALUE)GET_ISEQ(), cd, adjusted_cd.cd.cc);
}

if (UNDEF_P(val)) {
Expand Down
2 changes: 1 addition & 1 deletion vm_args.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1190,7 +1190,7 @@ vm_caller_setup_fwd_args(const rb_execution_context_t *ec, rb_control_frame_t *r
);

adjusted_cd->cd.ci = adjusted_ci;
adjusted_cd->cd.cc = cd->cc;
adjusted_cd->cd.cc = vm_cd_cc_load(cd);
adjusted_cd->caller_ci = caller_ci;

return bh;
Expand Down
25 changes: 23 additions & 2 deletions vm_callinfo.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -434,7 +434,7 @@ vm_cc_call(const struct rb_callcache *cc)
VM_ASSERT(cc->call_ != NULL);
VM_ASSERT(cc->klass != Qundef || !vm_cc_markable(cc));
VM_ASSERT(cc_check_class(cc->klass));
return cc->call_;
return (vm_call_handler)rbimpl_atomic_ptr_load((void **)&cc->call_, RBIMPL_ATOMIC_RELAXED);
}

static inline void
Expand Down Expand Up @@ -484,7 +484,7 @@ vm_cc_call_set(const struct rb_callcache *cc, vm_call_handler call)
{
VM_ASSERT(IMEMO_TYPE_P(cc, imemo_callcache));
VM_ASSERT(cc != vm_cc_empty());
*(vm_call_handler *)&cc->call_ = call;
rbimpl_atomic_ptr_store((volatile void **)&cc->call_, (void *)call, RBIMPL_ATOMIC_RELAXED);
}

static inline void
Expand Down Expand Up @@ -578,6 +578,27 @@ struct rb_call_data {
const struct rb_callcache *cc;
};

static inline const struct rb_callcache *
vm_cd_cc_load(const struct rb_call_data *cd)
{
return rbimpl_atomic_ptr_load((void **)&cd->cc, RBIMPL_ATOMIC_ACQUIRE);
}

static inline void
vm_cd_cc_store_raw(struct rb_call_data *cd, const struct rb_callcache *cc)
{
rbimpl_atomic_ptr_store((volatile void **)&cd->cc, (void *)cc, RBIMPL_ATOMIC_RELEASE);
}

static inline void
vm_cd_cc_store(VALUE owner, struct rb_call_data *cd, const struct rb_callcache *cc)
{
RUBY_ASSERT(vm_cc_markable(cc));

vm_cd_cc_store_raw(cd, cc);
RB_OBJ_WRITTEN(owner, Qundef, cc);
}

struct rb_class_cc_entries {
#if VM_CHECK_MODE > 0
VALUE debug_sig;
Expand Down
32 changes: 17 additions & 15 deletions vm_insnhelper.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2257,17 +2257,18 @@ static const struct rb_callcache *
vm_search_method_slowpath0(VALUE cd_owner, struct rb_call_data *cd, VALUE klass)
{
#if USE_DEBUG_COUNTER
const struct rb_callcache *old_cc = cd->cc;
const struct rb_callcache *old_cc = vm_cd_cc_load(cd);
#endif

const struct rb_callcache *cc = rb_vm_search_method_slowpath(cd->ci, klass);

#if OPT_INLINE_METHOD_CACHE
cd->cc = cc;

const struct rb_callcache *empty_cc = &vm_empty_cc;
if (cd_owner && cc != empty_cc) {
RB_OBJ_WRITTEN(cd_owner, Qundef, cc);
vm_cd_cc_store(cd_owner, cd, cc);
}
else {
vm_cd_cc_store_raw(cd, cc);
}

#if USE_DEBUG_COUNTER
Expand Down Expand Up @@ -2301,7 +2302,7 @@ ALWAYS_INLINE(static const struct rb_callcache *vm_search_method_fastpath(VALUE
static const struct rb_callcache *
vm_search_method_fastpath(VALUE cd_owner, struct rb_call_data *cd, VALUE klass)
{
const struct rb_callcache *cc = cd->cc;
const struct rb_callcache *cc = vm_cd_cc_load(cd);

#if OPT_INLINE_METHOD_CACHE
if (LIKELY(vm_cc_class_check(cc, klass))) {
Expand Down Expand Up @@ -4631,9 +4632,9 @@ vm_call_refined(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_c
const rb_callable_method_entry_t *ref_cme = search_refined_method(ec, cfp, calling);

if (ref_cme) {
if (calling->cd->cc) {
if (vm_cd_cc_load(calling->cd)) {
const struct rb_callcache *cc = calling->cc = vm_cc_new(vm_cc_cme(calling->cc)->defined_class, ref_cme, vm_call_general, cc_type_refinement);
RB_OBJ_WRITE(cfp->iseq, &calling->cd->cc, cc);
vm_cd_cc_store((VALUE)cfp->iseq, (struct rb_call_data *)calling->cd, cc);
return vm_call_method(ec, cfp, calling);
}
else {
Expand Down Expand Up @@ -5105,7 +5106,7 @@ vm_search_super_method(const rb_control_frame_t *reg_cfp, struct rb_call_data *c
if (!klass) {
/* bound instance method of module */
cc = vm_cc_new(klass, NULL, vm_call_method_missing, cc_type_super);
RB_OBJ_WRITE(reg_cfp->iseq, &cd->cc, cc);
vm_cd_cc_store((VALUE)reg_cfp->iseq, cd, cc);
}
else {
cc = vm_search_method_fastpath((VALUE)reg_cfp->iseq, cd, klass);
Expand All @@ -5114,16 +5115,17 @@ vm_search_super_method(const rb_control_frame_t *reg_cfp, struct rb_call_data *c
// define_method can cache for different method id
if (cached_cme == NULL) {
// empty_cc_for_super is not markable object
cd->cc = empty_cc_for_super();
vm_cd_cc_store_raw(cd, empty_cc_for_super());
}
else if (cached_cme->called_id != mid) {
const rb_callable_method_entry_t *cme = rb_callable_method_entry(klass, mid);
if (cme) {
cc = vm_cc_new(klass, cme, vm_call_super_method, cc_type_super);
RB_OBJ_WRITE(reg_cfp->iseq, &cd->cc, cc);
vm_cd_cc_store((VALUE)reg_cfp->iseq, cd, cc);
}
else {
cd->cc = cc = empty_cc_for_super();
cc = empty_cc_for_super();
vm_cd_cc_store_raw(cd, cc);
}
}
else {
Expand Down Expand Up @@ -6082,8 +6084,8 @@ rb_vm_send(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, CALL_DATA cd

val = vm_sendish(ec, GET_CFP(), &adjusted_cd.cd, bh, mexp_search_method);

if (cd->cc != adjusted_cd.cd.cc && vm_cc_markable(adjusted_cd.cd.cc)) {
RB_OBJ_WRITE(GET_ISEQ(), &cd->cc, adjusted_cd.cd.cc);
if (vm_cd_cc_load(cd) != adjusted_cd.cd.cc && vm_cc_markable(adjusted_cd.cd.cc)) {
vm_cd_cc_store((VALUE)GET_ISEQ(), cd, adjusted_cd.cd.cc);
}
}
else {
Expand Down Expand Up @@ -6120,8 +6122,8 @@ rb_vm_invokesuper(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, CALL_

val = vm_sendish(ec, GET_CFP(), &adjusted_cd.cd, bh, mexp_search_super);

if (cd->cc != adjusted_cd.cd.cc && vm_cc_markable(adjusted_cd.cd.cc)) {
RB_OBJ_WRITE(GET_ISEQ(), &cd->cc, adjusted_cd.cd.cc);
if (vm_cd_cc_load(cd) != adjusted_cd.cd.cc && vm_cc_markable(adjusted_cd.cd.cc)) {
vm_cd_cc_store((VALUE)GET_ISEQ(), cd, adjusted_cd.cd.cc);
}
}
else {
Expand Down