From c27bc9c6a00eb083eb9da79315ef596e8e15679f Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Wed, 3 Jan 2024 15:59:37 -0500 Subject: [PATCH 1/4] Memory leak when duplicating identhash [Bug #20145] Before this commit, both copy_compare_by_id and hash_copy will create a copy of the ST table, so the ST table created in copy_compare_by_id will be leaked. h = { 1 => 2 }.compare_by_identity 10.times do 1_000_000.times do h.select { false } end puts `ps -o rss= -p #{$$}` end Before: 110736 204352 300272 395520 460704 476736 542000 604704 682624 770528 After: 15504 16048 16144 16256 16320 16320 16752 16752 16752 16752 --- hash.c | 10 +++++++++- test/ruby/test_hash.rb | 10 ++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/hash.c b/hash.c index 78e9d9a2d60c8b..b15d856ee19c36 100644 --- a/hash.c +++ b/hash.c @@ -1557,7 +1557,15 @@ hash_copy(VALUE ret, VALUE hash) static VALUE hash_dup_with_compare_by_id(VALUE hash) { - return hash_copy(copy_compare_by_id(rb_hash_new(), hash), hash); + VALUE dup = hash_alloc_flags(rb_cHash, 0, Qnil, RHASH_ST_TABLE_P(hash)); + if (RHASH_ST_TABLE_P(hash)) { + RHASH_SET_ST_FLAG(dup); + } + else { + RHASH_UNSET_ST_FLAG(dup); + } + + return hash_copy(dup, hash); } static VALUE diff --git a/test/ruby/test_hash.rb b/test/ruby/test_hash.rb index 70daea0ef19150..c72b256bab39aa 100644 --- a/test/ruby/test_hash.rb +++ b/test/ruby/test_hash.rb @@ -1458,6 +1458,16 @@ def test_compare_by_identity assert_predicate(h.dup, :compare_by_identity?, bug8703) end + def test_compare_by_identy_memory_leak + assert_no_memory_leak([], "", "#{<<~"begin;"}\n#{<<~'end;'}", "[Bug #20145]", rss: true) + begin; + h = { 1 => 2 }.compare_by_identity + 1_000_000.times do + h.select { false } + end + end; + end + def test_same_key bug9646 = '[ruby-dev:48047] [Bug #9646] Infinite loop at Hash#each' h = @cls[a=[], 1] From 256350b765a51ecd9595dc05bf2ddebb8780b87f Mon Sep 17 00:00:00 2001 From: John Hawthorn Date: Thu, 11 Jan 2024 11:57:03 -0800 Subject: [PATCH 2/4] Avoid reading unused lvars in Primitive.cexpr Previously on builds with optimizations disabled, this could result in an out of bounds read. When we had all of: * built with -O0 * Leaf builtin * Primitive.mandatory_only * "no args builtin", called by vm_call_single_noarg_inline_builti * The stack is escaped to the heap via binding or a proc This is because mk_builtin_loader generated reads for all locals regardless of whether they were used and in the case we generated a mandatory_only iseq that would include more variables than were actually available. On optimized builds, the invalid accesses would be optimized away, and this also was often unnoticed as the invalid access would just hit another part of the stack unless it had been escaped to the heap. The fix here is imperfect, as this could have false positives, but since Primitive.cexpr! is only available within the cruby codebase itself that's probably fine as a proper fix would be much more challenging (the only false positives we found were in rjit.rb). Fixes [Bug #20178] Co-authored-by: Adam Hess --- bootstraptest/test_method.rb | 9 +++++++++ tool/mk_builtin_loader.rb | 6 ++++++ 2 files changed, 15 insertions(+) diff --git a/bootstraptest/test_method.rb b/bootstraptest/test_method.rb index 04c9eb2d11425e..964bf39d98b801 100644 --- a/bootstraptest/test_method.rb +++ b/bootstraptest/test_method.rb @@ -1190,3 +1190,12 @@ def test2 o, args, block test2 o1, [], block $result.join } + +assert_equal 'ok', %q{ + def foo + binding + ["ok"].first + end + foo + foo +}, '[Bug #20178]' diff --git a/tool/mk_builtin_loader.rb b/tool/mk_builtin_loader.rb index 5ab427ca9b2931..95600e6a3bdc55 100644 --- a/tool/mk_builtin_loader.rb +++ b/tool/mk_builtin_loader.rb @@ -263,11 +263,17 @@ def collect_iseq iseq_ary def generate_cexpr(ofile, lineno, line_file, body_lineno, text, locals, func_name) f = StringIO.new + + # Avoid generating fetches of lvars we don't need. This is imperfect as it + # will match text inside strings or other false positives. + local_candidates = text.scan(/[a-zA-Z_][a-zA-Z0-9_]*/) + f.puts '{' lineno += 1 # locals is nil outside methods locals&.reverse_each&.with_index{|param, i| next unless Symbol === param + next unless local_candidates.include?(param.to_s) f.puts "MAYBE_UNUSED(const VALUE) #{param} = rb_vm_lvar(ec, #{-3 - i});" lineno += 1 } From dff080dc4159328c17ef21123b20a5f428b8e404 Mon Sep 17 00:00:00 2001 From: Adam Hess Date: Thu, 4 Jan 2024 10:43:49 -0800 Subject: [PATCH 3/4] Free pthread_attr after setting up the thread [bug #20149] --- ext/socket/raddrinfo.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/ext/socket/raddrinfo.c b/ext/socket/raddrinfo.c index 9747f940369967..ceaac031a2b1ec 100644 --- a/ext/socket/raddrinfo.c +++ b/ext/socket/raddrinfo.c @@ -511,6 +511,11 @@ rb_getaddrinfo(const char *hostp, const char *portp, const struct addrinfo *hint } pthread_detach(th); + int r; + if ((r = pthread_attr_destroy(&attr)) != 0) { + rb_bug_errno("pthread_attr_destroy", r); + } + rb_thread_call_without_gvl2(wait_getaddrinfo, arg, cancel_getaddrinfo, arg); int need_free = 0; @@ -732,12 +737,17 @@ rb_getnameinfo(const struct sockaddr *sa, socklen_t salen, #endif pthread_t th; - if (do_pthread_create(&th, 0, do_getnameinfo, arg) != 0) { + if (do_pthread_create(&th, &attr, do_getnameinfo, arg) != 0) { free_getnameinfo_arg(arg); return EAI_AGAIN; } pthread_detach(th); + int r; + if ((r = pthread_attr_destroy(&attr)) != 0) { + rb_bug_errno("pthread_attr_destroy", r); + } + rb_thread_call_without_gvl2(wait_getnameinfo, arg, cancel_getnameinfo, arg); int need_free = 0; From a4890e4b90cd0433ab880363b36ce362c0d538d9 Mon Sep 17 00:00:00 2001 From: Yusuke Endoh Date: Wed, 10 Jan 2024 19:52:53 +0900 Subject: [PATCH 4/4] Remove setaffinity of pthread for getaddrinfo It looks like `sched_getcpu(3)` returns a strange number on some (virtual?) environments. I decided to remove the setaffinity mechanism because the performance does not appear to degrade on a quick benchmark even if removed. [Bug #20172] --- ext/socket/extconf.rb | 2 -- ext/socket/raddrinfo.c | 48 ++++-------------------------------------- 2 files changed, 4 insertions(+), 46 deletions(-) diff --git a/ext/socket/extconf.rb b/ext/socket/extconf.rb index 544bed5298cfc6..4e8536fc606bc0 100644 --- a/ext/socket/extconf.rb +++ b/ext/socket/extconf.rb @@ -706,8 +706,6 @@ def %(s) s || self end have_func("pthread_create") have_func("pthread_detach") - have_func("pthread_attr_setaffinity_np") - have_func("sched_getcpu") $VPATH << '$(topdir)' << '$(top_srcdir)' create_makefile("socket") diff --git a/ext/socket/raddrinfo.c b/ext/socket/raddrinfo.c index ceaac031a2b1ec..560312741f6820 100644 --- a/ext/socket/raddrinfo.c +++ b/ext/socket/raddrinfo.c @@ -461,7 +461,7 @@ cancel_getaddrinfo(void *ptr) } static int -do_pthread_create(pthread_t *th, const pthread_attr_t *attr, void *(*start_routine) (void *), void *arg) +do_pthread_create(pthread_t *th, void *(*start_routine) (void *), void *arg) { int limit = 3, ret; do { @@ -469,7 +469,7 @@ do_pthread_create(pthread_t *th, const pthread_attr_t *attr, void *(*start_routi // // https://bugs.openjdk.org/browse/JDK-8268605 // https://github.com/openjdk/jdk/commit/e35005d5ce383ddd108096a3079b17cb0bcf76f1 - ret = pthread_create(th, attr, start_routine, arg); + ret = pthread_create(th, 0, start_routine, arg); } while (ret == EAGAIN && limit-- > 0); return ret; } @@ -489,33 +489,13 @@ rb_getaddrinfo(const char *hostp, const char *portp, const struct addrinfo *hint return EAI_MEMORY; } - pthread_attr_t attr; - if (pthread_attr_init(&attr) != 0) { - free_getaddrinfo_arg(arg); - return EAI_AGAIN; - } -#if defined(HAVE_PTHREAD_ATTR_SETAFFINITY_NP) && defined(HAVE_SCHED_GETCPU) - cpu_set_t tmp_cpu_set; - CPU_ZERO(&tmp_cpu_set); - int cpu = sched_getcpu(); - if (cpu < CPU_SETSIZE) { - CPU_SET(cpu, &tmp_cpu_set); - pthread_attr_setaffinity_np(&attr, sizeof(cpu_set_t), &tmp_cpu_set); - } -#endif - pthread_t th; - if (do_pthread_create(&th, &attr, do_getaddrinfo, arg) != 0) { + if (do_pthread_create(&th, do_getaddrinfo, arg) != 0) { free_getaddrinfo_arg(arg); return EAI_AGAIN; } pthread_detach(th); - int r; - if ((r = pthread_attr_destroy(&attr)) != 0) { - rb_bug_errno("pthread_attr_destroy", r); - } - rb_thread_call_without_gvl2(wait_getaddrinfo, arg, cancel_getaddrinfo, arg); int need_free = 0; @@ -721,33 +701,13 @@ rb_getnameinfo(const struct sockaddr *sa, socklen_t salen, return EAI_MEMORY; } - pthread_attr_t attr; - if (pthread_attr_init(&attr) != 0) { - free_getnameinfo_arg(arg); - return EAI_AGAIN; - } -#if defined(HAVE_PTHREAD_ATTR_SETAFFINITY_NP) && defined(HAVE_SCHED_GETCPU) - cpu_set_t tmp_cpu_set; - CPU_ZERO(&tmp_cpu_set); - int cpu = sched_getcpu(); - if (cpu < CPU_SETSIZE) { - CPU_SET(cpu, &tmp_cpu_set); - pthread_attr_setaffinity_np(&attr, sizeof(cpu_set_t), &tmp_cpu_set); - } -#endif - pthread_t th; - if (do_pthread_create(&th, &attr, do_getnameinfo, arg) != 0) { + if (do_pthread_create(&th, do_getnameinfo, arg) != 0) { free_getnameinfo_arg(arg); return EAI_AGAIN; } pthread_detach(th); - int r; - if ((r = pthread_attr_destroy(&attr)) != 0) { - rb_bug_errno("pthread_attr_destroy", r); - } - rb_thread_call_without_gvl2(wait_getnameinfo, arg, cancel_getnameinfo, arg); int need_free = 0;