From afa1eff7730b8e00326fd15d0ec6d46e2dbba27d Mon Sep 17 00:00:00 2001 From: John Hawthorn Date: Fri, 11 Apr 2025 23:48:30 -0700 Subject: [PATCH 01/11] Eagerly initialize dtoa instead of using atomics --- eval.c | 2 ++ internal/missing.h | 3 +++ missing/dtoa.c | 50 +++++++++++++++++----------------------------- 3 files changed, 23 insertions(+), 32 deletions(-) diff --git a/eval.c b/eval.c index 6ee99359f9c82b..33dbf036d7536a 100644 --- a/eval.c +++ b/eval.c @@ -30,6 +30,7 @@ #include "internal/object.h" #include "internal/thread.h" #include "internal/variable.h" +#include "internal/missing.h" #include "ruby/fiber/scheduler.h" #include "iseq.h" #include "probes.h" @@ -77,6 +78,7 @@ ruby_setup(void) prctl(PR_SET_THP_DISABLE, 1, 0, 0, 0); #endif Init_BareVM(); + ruby_init_dtoa(); rb_vm_encoded_insn_data_table_init(); Init_vm_objects(); Init_fstring_table(); diff --git a/internal/missing.h b/internal/missing.h index 6ca508c8f9b543..fa9538d3b8ef56 100644 --- a/internal/missing.h +++ b/internal/missing.h @@ -16,4 +16,7 @@ extern void ruby_init_setproctitle(int argc, char *argv[]); extern void ruby_free_proctitle(void); #endif +/* missing/dtoa.c */ +void ruby_init_dtoa(void); + #endif /* INTERNAL_MISSING_H */ diff --git a/missing/dtoa.c b/missing/dtoa.c index 8859fcfa44f9fb..7fbfd2e81ab2ef 100644 --- a/missing/dtoa.c +++ b/missing/dtoa.c @@ -501,7 +501,6 @@ extern double rnd_prod(double, double), rnd_quot(double, double); #define Kmax 15 struct Bigint { - struct Bigint *next; int k, maxwds, sign, wds; ULong x[1]; }; @@ -800,13 +799,13 @@ mult(Bigint *a, Bigint *b) return c; } -static Bigint *p5s; +#define MAX_P5 8 +static Bigint *p5s_static[MAX_P5]; static Bigint * pow5mult(Bigint *b, int k) { - Bigint *b1, *p5, *p51; - Bigint *p5tmp; + Bigint *b1, *p5, **p5s; int i; static const int p05[3] = { 5, 25, 125 }; @@ -815,42 +814,20 @@ pow5mult(Bigint *b, int k) if (!(k >>= 2)) return b; - if (!(p5 = p5s)) { - /* first time */ - ACQUIRE_DTOA_LOCK(1); - if (!(p5 = p5s)) { - p5 = i2b(625); - p5->next = 0; - p5tmp = ATOMIC_PTR_CAS(p5s, NULL, p5); - if (UNLIKELY(p5tmp)) { - Bfree(p5); - p5 = p5tmp; - } - } - FREE_DTOA_LOCK(1); - } + p5s = &p5s_static[0]; for (;;) { + assert(p5s < &p5s_static[MAX_P5]); + p5 = *p5s++; + assert(p5); + if (k & 1) { b1 = mult(b, p5); Bfree(b); b = b1; } + if (!(k >>= 1)) break; - if (!(p51 = p5->next)) { - ACQUIRE_DTOA_LOCK(1); - if (!(p51 = p5->next)) { - p51 = mult(p5,p5); - p51->next = 0; - p5tmp = ATOMIC_PTR_CAS(p5->next, NULL, p51); - if (UNLIKELY(p5tmp)) { - Bfree(p51); - p51 = p5tmp; - } - } - FREE_DTOA_LOCK(1); - } - p5 = p51; } return b; } @@ -3380,6 +3357,15 @@ hdtoa(double d, const char *xdigs, int ndigits, int *decpt, int *sign, char **rv return (s0); } +void ruby_init_dtoa(void) { + Bigint *p5 = i2b(625); + p5s_static[0] = p5; + for (int i = 1; i < MAX_P5; i++) { + p5 = mult(p5,p5); + p5s_static[i] = p5; + } +} + #ifdef __cplusplus #if 0 { /* satisfy cc-mode */ From 0371827c248888a74891689b3740b6516837c706 Mon Sep 17 00:00:00 2001 From: John Hawthorn Date: Fri, 11 Apr 2025 23:49:48 -0700 Subject: [PATCH 02/11] Extract MAXWDS --- missing/dtoa.c | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/missing/dtoa.c b/missing/dtoa.c index 7fbfd2e81ab2ef..fe42caf78da197 100644 --- a/missing/dtoa.c +++ b/missing/dtoa.c @@ -507,6 +507,15 @@ struct Bigint { typedef struct Bigint Bigint; +static inline int MAXWDS(Bigint *b) { + RUBY_ASSERT_ALWAYS(b->maxwds == (1 << b->k)); + if (b->maxwds != (1 << b->k)) { + rb_bug("wtf"); + } + //return b->maxwds; + return (1 << b->k); +} + static Bigint * Balloc(int k) { @@ -569,7 +578,7 @@ multadd(Bigint *b, int m, int a) /* multiply by m and add a */ #endif } while (++i < wds); if (carry) { - if (wds >= b->maxwds) { + if (wds >= MAXWDS(b)) { b1 = Balloc(b->k+1); Bcopy(b1, b); Bfree(b); @@ -723,7 +732,7 @@ mult(Bigint *a, Bigint *b) wa = a->wds; wb = b->wds; wc = wa + wb; - if (wc > a->maxwds) + if (wc > MAXWDS(a)) k++; c = Balloc(k); for (x = c->x, xa = x + wc; x < xa; x++) @@ -846,7 +855,7 @@ lshift(Bigint *b, int k) #endif k1 = b->k; n1 = n + b->wds + 1; - for (i = b->maxwds; n1 > i; i <<= 1) + for (i = MAXWDS(b); n1 > i; i <<= 1) k1++; b1 = Balloc(k1); x1 = b1->x; From 0ead7384d892c323e2a9b04022b5855ffcf2f3a9 Mon Sep 17 00:00:00 2001 From: John Hawthorn Date: Sat, 12 Apr 2025 16:01:51 -0700 Subject: [PATCH 03/11] More sensible Bcopy --- missing/dtoa.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/missing/dtoa.c b/missing/dtoa.c index fe42caf78da197..cf44bf42114cfc 100644 --- a/missing/dtoa.c +++ b/missing/dtoa.c @@ -536,8 +536,12 @@ Bfree(Bigint *v) FREE(v); } -#define Bcopy(x,y) memcpy((char *)&(x)->sign, (char *)&(y)->sign, \ -(y)->wds*sizeof(Long) + 2*sizeof(int)) +// Copy everything except k and maxwds +static void Bcopy(Bigint *dst, Bigint *src) { + dst->sign = src->sign; + dst->wds = src->wds; + memcpy(dst->x, src->x, sizeof(Long) * src->wds); +} static Bigint * multadd(Bigint *b, int m, int a) /* multiply by m and add a */ From 123a89d88cec88d94f77ccc6f0dfcd4e1ec8014b Mon Sep 17 00:00:00 2001 From: John Hawthorn Date: Thu, 17 Apr 2025 23:47:11 -0700 Subject: [PATCH 04/11] Remove maxwds --- missing/dtoa.c | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/missing/dtoa.c b/missing/dtoa.c index cf44bf42114cfc..fb05ec9f352cf5 100644 --- a/missing/dtoa.c +++ b/missing/dtoa.c @@ -501,18 +501,13 @@ extern double rnd_prod(double, double), rnd_quot(double, double); #define Kmax 15 struct Bigint { - int k, maxwds, sign, wds; + int k, sign, wds; ULong x[1]; }; typedef struct Bigint Bigint; static inline int MAXWDS(Bigint *b) { - RUBY_ASSERT_ALWAYS(b->maxwds == (1 << b->k)); - if (b->maxwds != (1 << b->k)) { - rb_bug("wtf"); - } - //return b->maxwds; return (1 << b->k); } @@ -525,7 +520,6 @@ Balloc(int k) x = 1 << k; rv = (Bigint *)MALLOC(sizeof(Bigint) + (x-1)*sizeof(ULong)); rv->k = k; - rv->maxwds = x; rv->sign = rv->wds = 0; return rv; } @@ -536,7 +530,7 @@ Bfree(Bigint *v) FREE(v); } -// Copy everything except k and maxwds +// Copy everything except k static void Bcopy(Bigint *dst, Bigint *src) { dst->sign = src->sign; dst->wds = src->wds; From 0b71a7e620580a2642033e0afbb15276a893b210 Mon Sep 17 00:00:00 2001 From: John Hawthorn Date: Thu, 17 Apr 2025 23:50:03 -0700 Subject: [PATCH 05/11] Shrink fields of Bigint --- missing/dtoa.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/missing/dtoa.c b/missing/dtoa.c index fb05ec9f352cf5..007c082adb4f9f 100644 --- a/missing/dtoa.c +++ b/missing/dtoa.c @@ -501,7 +501,9 @@ extern double rnd_prod(double, double), rnd_quot(double, double); #define Kmax 15 struct Bigint { - int k, sign, wds; + unsigned char k; + unsigned short wds; + unsigned char sign : 1; ULong x[1]; }; From ad889ce71164472188b1bbafbb0bb10e871dc01e Mon Sep 17 00:00:00 2001 From: John Hawthorn Date: Fri, 18 Apr 2025 00:15:18 -0700 Subject: [PATCH 06/11] Pass DTOA state through everything --- missing/dtoa.c | 179 ++++++++++++++++++++++++++----------------------- 1 file changed, 96 insertions(+), 83 deletions(-) diff --git a/missing/dtoa.c b/missing/dtoa.c index 007c082adb4f9f..060f2680291c2c 100644 --- a/missing/dtoa.c +++ b/missing/dtoa.c @@ -513,8 +513,15 @@ static inline int MAXWDS(Bigint *b) { return (1 << b->k); } +struct dtoa_state { +}; + +#define DTOA_STATE_INIT struct dtoa_state __dtoa_state, *_dtoa_state = &__dtoa_state; +#define DTOA_STATE_SIG struct dtoa_state *_dtoa_state +#define DTOA_STATE_CALL (_dtoa_state) + static Bigint * -Balloc(int k) +Balloc(DTOA_STATE_SIG, int k) { int x; Bigint *rv; @@ -527,7 +534,7 @@ Balloc(int k) } static void -Bfree(Bigint *v) +Bfree(DTOA_STATE_SIG, Bigint *v) { FREE(v); } @@ -540,7 +547,7 @@ static void Bcopy(Bigint *dst, Bigint *src) { } static Bigint * -multadd(Bigint *b, int m, int a) /* multiply by m and add a */ +multadd(DTOA_STATE_SIG, Bigint *b, int m, int a) /* multiply by m and add a */ { int i, wds; ULong *x; @@ -579,9 +586,9 @@ multadd(Bigint *b, int m, int a) /* multiply by m and add a */ } while (++i < wds); if (carry) { if (wds >= MAXWDS(b)) { - b1 = Balloc(b->k+1); + b1 = Balloc(DTOA_STATE_CALL, b->k+1); Bcopy(b1, b); - Bfree(b); + Bfree(DTOA_STATE_CALL, b); b = b1; } b->x[wds++] = (ULong)carry; @@ -591,7 +598,7 @@ multadd(Bigint *b, int m, int a) /* multiply by m and add a */ } static Bigint * -s2b(const char *s, int nd0, int nd, ULong y9) +s2b(DTOA_STATE_SIG, const char *s, int nd0, int nd, ULong y9) { Bigint *b; int i, k; @@ -600,11 +607,11 @@ s2b(const char *s, int nd0, int nd, ULong y9) x = (nd + 8) / 9; for (k = 0, y = 1; x > y; y <<= 1, k++) ; #ifdef Pack_32 - b = Balloc(k); + b = Balloc(DTOA_STATE_CALL, k); b->x[0] = y9; b->wds = 1; #else - b = Balloc(k+1); + b = Balloc(DTOA_STATE_CALL, k+1); b->x[0] = y9 & 0xffff; b->wds = (b->x[1] = y9 >> 16) ? 2 : 1; #endif @@ -613,14 +620,14 @@ s2b(const char *s, int nd0, int nd, ULong y9) if (9 < nd0) { s += 9; do { - b = multadd(b, 10, *s++ - '0'); + b = multadd(DTOA_STATE_CALL, b, 10, *s++ - '0'); } while (++i < nd0); s++; } else s += 10; for (; i < nd; i++) - b = multadd(b, 10, *s++ - '0'); + b = multadd(DTOA_STATE_CALL, b, 10, *s++ - '0'); return b; } @@ -697,18 +704,18 @@ lo0bits(ULong *y) } static Bigint * -i2b(int i) +i2b(DTOA_STATE_SIG, int i) { Bigint *b; - b = Balloc(1); + b = Balloc(DTOA_STATE_CALL, 1); b->x[0] = i; b->wds = 1; return b; } static Bigint * -mult(Bigint *a, Bigint *b) +mult(DTOA_STATE_SIG, Bigint *a, Bigint *b) { Bigint *c; int k, wa, wb, wc; @@ -734,7 +741,7 @@ mult(Bigint *a, Bigint *b) wc = wa + wb; if (wc > MAXWDS(a)) k++; - c = Balloc(k); + c = Balloc(DTOA_STATE_CALL, k); for (x = c->x, xa = x + wc; x < xa; x++) *x = 0; xa = a->x; @@ -812,14 +819,14 @@ mult(Bigint *a, Bigint *b) static Bigint *p5s_static[MAX_P5]; static Bigint * -pow5mult(Bigint *b, int k) +pow5mult(DTOA_STATE_SIG, Bigint *b, int k) { Bigint *b1, *p5, **p5s; int i; static const int p05[3] = { 5, 25, 125 }; if ((i = k & 3) != 0) - b = multadd(b, p05[i-1], 0); + b = multadd(DTOA_STATE_CALL, b, p05[i-1], 0); if (!(k >>= 2)) return b; @@ -830,8 +837,8 @@ pow5mult(Bigint *b, int k) assert(p5); if (k & 1) { - b1 = mult(b, p5); - Bfree(b); + b1 = mult(DTOA_STATE_CALL, b, p5); + Bfree(DTOA_STATE_CALL, b); b = b1; } @@ -842,7 +849,7 @@ pow5mult(Bigint *b, int k) } static Bigint * -lshift(Bigint *b, int k) +lshift(DTOA_STATE_SIG, Bigint *b, int k) { int i, k1, n, n1; Bigint *b1; @@ -857,7 +864,7 @@ lshift(Bigint *b, int k) n1 = n + b->wds + 1; for (i = MAXWDS(b); n1 > i; i <<= 1) k1++; - b1 = Balloc(k1); + b1 = Balloc(DTOA_STATE_CALL, k1); x1 = b1->x; for (i = 0; i < n; i++) *x1++ = 0; @@ -891,7 +898,7 @@ lshift(Bigint *b, int k) *x1++ = *x++; } while (x < xe); b1->wds = n1 - 1; - Bfree(b); + Bfree(DTOA_STATE_CALL, b); return b1; } @@ -924,9 +931,9 @@ cmp(Bigint *a, Bigint *b) return 0; } -NO_SANITIZE("unsigned-integer-overflow", static Bigint * diff(Bigint *a, Bigint *b)); +NO_SANITIZE("unsigned-integer-overflow", static Bigint * diff(DTOA_STATE_SIG, Bigint *a, Bigint *b)); static Bigint * -diff(Bigint *a, Bigint *b) +diff(DTOA_STATE_SIG, Bigint *a, Bigint *b) { Bigint *c; int i, wa, wb; @@ -942,7 +949,7 @@ diff(Bigint *a, Bigint *b) i = cmp(a,b); if (!i) { - c = Balloc(0); + c = Balloc(DTOA_STATE_CALL, 0); c->wds = 1; c->x[0] = 0; return c; @@ -955,7 +962,7 @@ diff(Bigint *a, Bigint *b) } else i = 0; - c = Balloc(a->k); + c = Balloc(DTOA_STATE_CALL, a->k); c->sign = i; wa = a->wds; xa = a->x; @@ -1115,7 +1122,7 @@ b2d(Bigint *a, int *e) } static Bigint * -d2b(double d_, int *e, int *bits) +d2b(DTOA_STATE_SIG, double d_, int *e, int *bits) { double_u d; Bigint *b; @@ -1137,9 +1144,9 @@ d2b(double d_, int *e, int *bits) #endif #ifdef Pack_32 - b = Balloc(1); + b = Balloc(DTOA_STATE_CALL, 1); #else - b = Balloc(2); + b = Balloc(DTOA_STATE_CALL, 2); #endif x = b->x; @@ -1429,6 +1436,8 @@ strtod(const char *s00, char **se) const char *s2; #endif + DTOA_STATE_INIT; + errno = 0; sign = nz0 = nz = 0; dval(rv) = 0.; @@ -1890,13 +1899,13 @@ strtod(const char *s00, char **se) /* Put digits into bd: true value = bd * 10^e */ - bd0 = s2b(s0, nd0, nd, y); + bd0 = s2b(DTOA_STATE_CALL, s0, nd0, nd, y); for (;;) { - bd = Balloc(bd0->k); + bd = Balloc(DTOA_STATE_CALL, bd0->k); Bcopy(bd, bd0); - bb = d2b(dval(rv), &bbe, &bbbits); /* rv = bb * 2^bbe */ - bs = i2b(1); + bb = d2b(DTOA_STATE_CALL, dval(rv), &bbe, &bbbits); /* rv = bb * 2^bbe */ + bs = i2b(DTOA_STATE_CALL, 1); if (e >= 0) { bb2 = bb5 = 0; @@ -1952,20 +1961,20 @@ strtod(const char *s00, char **se) bs2 -= i; } if (bb5 > 0) { - bs = pow5mult(bs, bb5); - bb1 = mult(bs, bb); - Bfree(bb); + bs = pow5mult(DTOA_STATE_CALL, bs, bb5); + bb1 = mult(DTOA_STATE_CALL, bs, bb); + Bfree(DTOA_STATE_CALL, bb); bb = bb1; } if (bb2 > 0) - bb = lshift(bb, bb2); + bb = lshift(DTOA_STATE_CALL, bb, bb2); if (bd5 > 0) - bd = pow5mult(bd, bd5); + bd = pow5mult(DTOA_STATE_CALL, bd, bd5); if (bd2 > 0) - bd = lshift(bd, bd2); + bd = lshift(DTOA_STATE_CALL, bd, bd2); if (bs2 > 0) - bs = lshift(bs, bs2); - delta = diff(bb, bd); + bs = lshift(DTOA_STATE_CALL, bs, bs2); + delta = diff(DTOA_STATE_CALL, bb, bd); dsign = delta->sign; delta->sign = 0; i = cmp(delta, bs); @@ -1997,7 +2006,7 @@ strtod(const char *s00, char **se) if (y) #endif { - delta = lshift(delta,Log2P); + delta = lshift(DTOA_STATE_CALL, delta,Log2P); if (cmp(delta, bs) <= 0) adj = -0.5; } @@ -2086,7 +2095,7 @@ strtod(const char *s00, char **se) #endif break; } - delta = lshift(delta,Log2P); + delta = lshift(DTOA_STATE_CALL, delta,Log2P); if (cmp(delta, bs) > 0) goto drop_down; break; @@ -2310,10 +2319,10 @@ strtod(const char *s00, char **se) } #endif cont: - Bfree(bb); - Bfree(bd); - Bfree(bs); - Bfree(delta); + Bfree(DTOA_STATE_CALL, bb); + Bfree(DTOA_STATE_CALL, bd); + Bfree(DTOA_STATE_CALL, bs); + Bfree(DTOA_STATE_CALL, delta); } #ifdef SET_INEXACT if (inexact) { @@ -2346,11 +2355,11 @@ strtod(const char *s00, char **se) } #endif retfree: - Bfree(bb); - Bfree(bd); - Bfree(bs); - Bfree(bd0); - Bfree(delta); + Bfree(DTOA_STATE_CALL, bb); + Bfree(DTOA_STATE_CALL, bd); + Bfree(DTOA_STATE_CALL, bs); + Bfree(DTOA_STATE_CALL, bd0); + Bfree(DTOA_STATE_CALL, delta); ret: if (se) *se = (char *)s; @@ -2605,6 +2614,8 @@ dtoa(double d_, int mode, int ndigits, int *decpt, int *sign, char **rve) int inexact, oldinexact; #endif + DTOA_STATE_INIT; + dval(d) = d_; #ifndef MULTIPLE_THREADS @@ -2660,7 +2671,7 @@ dtoa(double d_, int mode, int ndigits, int *decpt, int *sign, char **rve) } #endif - b = d2b(dval(d), &be, &bbits); + b = d2b(DTOA_STATE_CALL, dval(d), &be, &bbits); #ifdef Sudden_Underflow i = (int)(word0(d) >> Exp_shift1 & (Exp_mask>>Exp_shift1)); #else @@ -2966,7 +2977,7 @@ dtoa(double d_, int mode, int ndigits, int *decpt, int *sign, char **rve) #endif b2 += i; s2 += i; - mhi = i2b(1); + mhi = i2b(DTOA_STATE_CALL, 1); } if (m2 > 0 && s2 > 0) { i = m2 < s2 ? m2 : s2; @@ -2977,20 +2988,20 @@ dtoa(double d_, int mode, int ndigits, int *decpt, int *sign, char **rve) if (b5 > 0) { if (leftright) { if (m5 > 0) { - mhi = pow5mult(mhi, m5); - b1 = mult(mhi, b); - Bfree(b); + mhi = pow5mult(DTOA_STATE_CALL, mhi, m5); + b1 = mult(DTOA_STATE_CALL, mhi, b); + Bfree(DTOA_STATE_CALL, b); b = b1; } if ((j = b5 - m5) != 0) - b = pow5mult(b, j); + b = pow5mult(DTOA_STATE_CALL, b, j); } else - b = pow5mult(b, b5); + b = pow5mult(DTOA_STATE_CALL, b, b5); } - S = i2b(1); + S = i2b(DTOA_STATE_CALL, 1); if (s5 > 0) - S = pow5mult(S, s5); + S = pow5mult(DTOA_STATE_CALL, S, s5); /* Check for special case that d is a normalized power of 2. */ @@ -3039,20 +3050,20 @@ dtoa(double d_, int mode, int ndigits, int *decpt, int *sign, char **rve) s2 += i; } if (b2 > 0) - b = lshift(b, b2); + b = lshift(DTOA_STATE_CALL, b, b2); if (s2 > 0) - S = lshift(S, s2); + S = lshift(DTOA_STATE_CALL, S, s2); if (k_check) { if (cmp(b,S) < 0) { k--; - b = multadd(b, 10, 0); /* we botched the k estimate */ + b = multadd(DTOA_STATE_CALL, b, 10, 0); /* we botched the k estimate */ if (leftright) - mhi = multadd(mhi, 10, 0); + mhi = multadd(DTOA_STATE_CALL, mhi, 10, 0); ilim = ilim1; } } if (ilim <= 0 && (mode == 3 || mode == 5)) { - if (ilim < 0 || cmp(b,S = multadd(S,5,0)) <= 0) { + if (ilim < 0 || cmp(b,S = multadd(DTOA_STATE_CALL, S,5,0)) <= 0) { /* no digits, fcvt style */ no_digits: k = -1 - ndigits; @@ -3065,7 +3076,7 @@ dtoa(double d_, int mode, int ndigits, int *decpt, int *sign, char **rve) } if (leftright) { if (m2 > 0) - mhi = lshift(mhi, m2); + mhi = lshift(DTOA_STATE_CALL, mhi, m2); /* Compute mlo -- check for special case * that d is a normalized power of 2. @@ -3073,9 +3084,9 @@ dtoa(double d_, int mode, int ndigits, int *decpt, int *sign, char **rve) mlo = mhi; if (spec_case) { - mhi = Balloc(mhi->k); + mhi = Balloc(DTOA_STATE_CALL, mhi->k); Bcopy(mhi, mlo); - mhi = lshift(mhi, Log2P); + mhi = lshift(DTOA_STATE_CALL, mhi, Log2P); } for (i = 1;;i++) { @@ -3084,9 +3095,9 @@ dtoa(double d_, int mode, int ndigits, int *decpt, int *sign, char **rve) * that will round to d? */ j = cmp(b, mlo); - delta = diff(S, mhi); + delta = diff(DTOA_STATE_CALL, S, mhi); j1 = delta->sign ? 1 : cmp(b, delta); - Bfree(delta); + Bfree(DTOA_STATE_CALL, delta); #ifndef ROUND_BIASED if (j1 == 0 && mode != 1 && !(word1(d) & 1) #ifdef Honor_FLT_ROUNDS @@ -3124,7 +3135,7 @@ dtoa(double d_, int mode, int ndigits, int *decpt, int *sign, char **rve) } #endif /*Honor_FLT_ROUNDS*/ if (j1 > 0) { - b = lshift(b, 1); + b = lshift(DTOA_STATE_CALL, b, 1); j1 = cmp(b, S); if ((j1 > 0 || (j1 == 0 && (dig & 1))) && dig++ == '9') goto round_9_up; @@ -3152,12 +3163,12 @@ dtoa(double d_, int mode, int ndigits, int *decpt, int *sign, char **rve) *s++ = dig; if (i == ilim) break; - b = multadd(b, 10, 0); + b = multadd(DTOA_STATE_CALL, b, 10, 0); if (mlo == mhi) - mlo = mhi = multadd(mhi, 10, 0); + mlo = mhi = multadd(DTOA_STATE_CALL, mhi, 10, 0); else { - mlo = multadd(mlo, 10, 0); - mhi = multadd(mhi, 10, 0); + mlo = multadd(DTOA_STATE_CALL, mlo, 10, 0); + mhi = multadd(DTOA_STATE_CALL, mhi, 10, 0); } } } @@ -3172,7 +3183,7 @@ dtoa(double d_, int mode, int ndigits, int *decpt, int *sign, char **rve) } if (i >= ilim) break; - b = multadd(b, 10, 0); + b = multadd(DTOA_STATE_CALL, b, 10, 0); } /* Round off last digit */ @@ -3183,7 +3194,7 @@ dtoa(double d_, int mode, int ndigits, int *decpt, int *sign, char **rve) case 2: goto roundoff; } #endif - b = lshift(b, 1); + b = lshift(DTOA_STATE_CALL, b, 1); j = cmp(b, S); if (j > 0 || (j == 0 && (dig & 1))) { roundoff: @@ -3201,11 +3212,11 @@ dtoa(double d_, int mode, int ndigits, int *decpt, int *sign, char **rve) } s++; ret: - Bfree(S); + Bfree(DTOA_STATE_CALL, S); if (mhi) { if (mlo && mlo != mhi) - Bfree(mlo); - Bfree(mhi); + Bfree(DTOA_STATE_CALL, mlo); + Bfree(DTOA_STATE_CALL, mhi); } ret1: #ifdef SET_INEXACT @@ -3219,7 +3230,7 @@ dtoa(double d_, int mode, int ndigits, int *decpt, int *sign, char **rve) else if (!oldinexact) clear_inexact(); #endif - Bfree(b); + Bfree(DTOA_STATE_CALL, b); *s = 0; *decpt = k + 1; if (rve) @@ -3367,10 +3378,12 @@ hdtoa(double d, const char *xdigs, int ndigits, int *decpt, int *sign, char **rv } void ruby_init_dtoa(void) { - Bigint *p5 = i2b(625); + DTOA_STATE_INIT; + + Bigint *p5 = i2b(DTOA_STATE_CALL, 625); p5s_static[0] = p5; for (int i = 1; i < MAX_P5; i++) { - p5 = mult(p5,p5); + p5 = mult(DTOA_STATE_CALL, p5,p5); p5s_static[i] = p5; } } From aacb3d2cc71b4568bd9d07eff2027f064606b2c3 Mon Sep 17 00:00:00 2001 From: John Hawthorn Date: Fri, 18 Apr 2025 00:17:35 -0700 Subject: [PATCH 07/11] Pack struct better --- missing/dtoa.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/missing/dtoa.c b/missing/dtoa.c index 060f2680291c2c..fe657307b1348d 100644 --- a/missing/dtoa.c +++ b/missing/dtoa.c @@ -501,8 +501,8 @@ extern double rnd_prod(double, double), rnd_quot(double, double); #define Kmax 15 struct Bigint { - unsigned char k; unsigned short wds; + unsigned char k; unsigned char sign : 1; ULong x[1]; }; From ed38b292c88f5409882e6f27484393059c1c55d3 Mon Sep 17 00:00:00 2001 From: John Hawthorn Date: Fri, 18 Apr 2025 00:48:03 -0700 Subject: [PATCH 08/11] On stack buffer for bigint allocations --- missing/dtoa.c | 46 ++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 38 insertions(+), 8 deletions(-) diff --git a/missing/dtoa.c b/missing/dtoa.c index fe657307b1348d..f6572e67e18c3f 100644 --- a/missing/dtoa.c +++ b/missing/dtoa.c @@ -503,7 +503,8 @@ extern double rnd_prod(double, double), rnd_quot(double, double); struct Bigint { unsigned short wds; unsigned char k; - unsigned char sign : 1; + unsigned int sign : 1; + unsigned int stackbuf_pos : 4; ULong x[1]; }; @@ -513,21 +514,42 @@ static inline int MAXWDS(Bigint *b) { return (1 << b->k); } +#define BIGINT_MEMSIZE(k) (sizeof(Bigint) + ((1 << k) - 1) * sizeof(ULong)) + +#define STACK_BIGINT_COUNT 4 +#define STACK_BIGINT_K 2 +#define STACK_BIGINT_SIZE (BIGINT_MEMSIZE(STACK_BIGINT_K)) +#define STACK_BIGINT_BUF_SIZE (STACK_BIGINT_SIZE * STACK_BIGINT_COUNT) + struct dtoa_state { + unsigned char available; + char buf[STACK_BIGINT_BUF_SIZE]; }; -#define DTOA_STATE_INIT struct dtoa_state __dtoa_state, *_dtoa_state = &__dtoa_state; -#define DTOA_STATE_SIG struct dtoa_state *_dtoa_state -#define DTOA_STATE_CALL (_dtoa_state) +#define DTOA_STATE_INIT struct dtoa_state _dtoa_state, *dtoa_state = &_dtoa_state; dtoa_state_init(dtoa_state); +#define DTOA_STATE_SIG struct dtoa_state *dtoa_state +#define DTOA_STATE_CALL (dtoa_state) + +static void +dtoa_state_init(DTOA_STATE_SIG) { + dtoa_state->available = (1 << (STACK_BIGINT_COUNT)) - 1; +} + +#include "internal/bits.h" static Bigint * Balloc(DTOA_STATE_SIG, int k) { - int x; Bigint *rv; - x = 1 << k; - rv = (Bigint *)MALLOC(sizeof(Bigint) + (x-1)*sizeof(ULong)); + if (dtoa_state->available && k <= STACK_BIGINT_K) { + int idx = ntz_int32(dtoa_state->available); + dtoa_state->available &= ~(1 << idx); + rv = (Bigint *)&dtoa_state->buf[idx * STACK_BIGINT_SIZE]; + } else { + rv = (Bigint *)MALLOC(BIGINT_MEMSIZE(k)); + } + rv->k = k; rv->sign = rv->wds = 0; return rv; @@ -536,7 +558,12 @@ Balloc(DTOA_STATE_SIG, int k) static void Bfree(DTOA_STATE_SIG, Bigint *v) { - FREE(v); + if ((uintptr_t)v >= (uintptr_t)dtoa_state->buf && (uintptr_t)v < ((uintptr_t)dtoa_state->buf + STACK_BIGINT_BUF_SIZE)) { + unsigned long idx = ((uintptr_t)v - (uintptr_t)dtoa_state->buf) / STACK_BIGINT_SIZE; + dtoa_state->available |= (1 << idx); + } else { + FREE(v); + } } // Copy everything except k @@ -3380,6 +3407,9 @@ hdtoa(double d, const char *xdigs, int ndigits, int *decpt, int *sign, char **rv void ruby_init_dtoa(void) { DTOA_STATE_INIT; + // HACK: Forbid stack allocation + dtoa_state->available = 0; + Bigint *p5 = i2b(DTOA_STATE_CALL, 625); p5s_static[0] = p5; for (int i = 1; i < MAX_P5; i++) { From 41eebea438bd7df6485b748a78878b0d88a7e50d Mon Sep 17 00:00:00 2001 From: John Hawthorn Date: Fri, 18 Apr 2025 03:45:13 -0700 Subject: [PATCH 09/11] Cleanup hack --- missing/dtoa.c | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/missing/dtoa.c b/missing/dtoa.c index f6572e67e18c3f..759ae245d31539 100644 --- a/missing/dtoa.c +++ b/missing/dtoa.c @@ -522,17 +522,25 @@ static inline int MAXWDS(Bigint *b) { #define STACK_BIGINT_BUF_SIZE (STACK_BIGINT_SIZE * STACK_BIGINT_COUNT) struct dtoa_state { + // a bitmap of available bignums unsigned char available; + + // memory buffer to store temporary bignums char buf[STACK_BIGINT_BUF_SIZE]; }; -#define DTOA_STATE_INIT struct dtoa_state _dtoa_state, *dtoa_state = &_dtoa_state; dtoa_state_init(dtoa_state); +#define DTOA_STATE_INIT(allow_stack) struct dtoa_state _dtoa_state, *dtoa_state = &_dtoa_state; dtoa_state_init(dtoa_state, allow_stack); #define DTOA_STATE_SIG struct dtoa_state *dtoa_state #define DTOA_STATE_CALL (dtoa_state) static void -dtoa_state_init(DTOA_STATE_SIG) { - dtoa_state->available = (1 << (STACK_BIGINT_COUNT)) - 1; +dtoa_state_init(DTOA_STATE_SIG, int allow_stack) { + if (allow_stack) { + dtoa_state->available = (1 << (STACK_BIGINT_COUNT)) - 1; + } + else { + dtoa_state->available = 0; + } } #include "internal/bits.h" @@ -1463,7 +1471,7 @@ strtod(const char *s00, char **se) const char *s2; #endif - DTOA_STATE_INIT; + DTOA_STATE_INIT(1); errno = 0; sign = nz0 = nz = 0; @@ -2641,7 +2649,7 @@ dtoa(double d_, int mode, int ndigits, int *decpt, int *sign, char **rve) int inexact, oldinexact; #endif - DTOA_STATE_INIT; + DTOA_STATE_INIT(1); dval(d) = d_; @@ -3405,10 +3413,7 @@ hdtoa(double d, const char *xdigs, int ndigits, int *decpt, int *sign, char **rv } void ruby_init_dtoa(void) { - DTOA_STATE_INIT; - - // HACK: Forbid stack allocation - dtoa_state->available = 0; + DTOA_STATE_INIT(0); Bigint *p5 = i2b(DTOA_STATE_CALL, 625); p5s_static[0] = p5; From e02cfa061ca3cc3fa92e5c2e6034f9f8dfc394bc Mon Sep 17 00:00:00 2001 From: John Hawthorn Date: Fri, 18 Apr 2025 03:50:08 -0700 Subject: [PATCH 10/11] Use int for wds again Truth is, I don't know how large this is --- missing/dtoa.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/missing/dtoa.c b/missing/dtoa.c index 759ae245d31539..38452518ca4729 100644 --- a/missing/dtoa.c +++ b/missing/dtoa.c @@ -501,10 +501,9 @@ extern double rnd_prod(double, double), rnd_quot(double, double); #define Kmax 15 struct Bigint { - unsigned short wds; + int wds; unsigned char k; - unsigned int sign : 1; - unsigned int stackbuf_pos : 4; + int sign : 1; ULong x[1]; }; From a0af6c2a50ef967779e7115f4823219fe853bac9 Mon Sep 17 00:00:00 2001 From: John Hawthorn Date: Thu, 1 May 2025 16:28:06 -0700 Subject: [PATCH 11/11] Update deps --- common.mk | 2 ++ 1 file changed, 2 insertions(+) diff --git a/common.mk b/common.mk index 4ae88f17649644..eebc0e953c4244 100644 --- a/common.mk +++ b/common.mk @@ -6821,6 +6821,7 @@ eval.$(OBJEXT): $(top_srcdir)/internal/hash.h eval.$(OBJEXT): $(top_srcdir)/internal/imemo.h eval.$(OBJEXT): $(top_srcdir)/internal/inits.h eval.$(OBJEXT): $(top_srcdir)/internal/io.h +eval.$(OBJEXT): $(top_srcdir)/internal/missing.h eval.$(OBJEXT): $(top_srcdir)/internal/object.h eval.$(OBJEXT): $(top_srcdir)/internal/sanitizers.h eval.$(OBJEXT): $(top_srcdir)/internal/serial.h @@ -18677,6 +18678,7 @@ transcode.$(OBJEXT): {$(VPATH)}transcode.c transcode.$(OBJEXT): {$(VPATH)}transcode_data.h util.$(OBJEXT): $(hdrdir)/ruby/ruby.h util.$(OBJEXT): $(top_srcdir)/internal/array.h +util.$(OBJEXT): $(top_srcdir)/internal/bits.h util.$(OBJEXT): $(top_srcdir)/internal/compilers.h util.$(OBJEXT): $(top_srcdir)/internal/imemo.h util.$(OBJEXT): $(top_srcdir)/internal/sanitizers.h