Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 261a5c3

Browse files
lukesandberglukesandberg
authored
[turbopack] mark rcstr! allocated Rcstr values as 'static' and stop refcounting them (#81994)
# Optimize RcStr with Static Atom Support ## What? For RcStr values allocated by the `rcstr!` macro we shouldn't bother reference counting them since they are pinned to a static memory location already. This uses a spare tag bit `0b10` to mark such strings as 'static' which allows us to to optimize the `clone` and `drop`. The only practical cost is an additional constant bit mask when reading the value as a pointer. Also: * add a fast path for `PartialEq<RcStr>` to handle identical values, this is useful especially for inline RcStr values but adds a fast path for refcounted strings as well. * compute the `hash` of `rcstr!` parameters at compile time by `const`-ifying the hash fn * allocate static storage for the `PrecomputeHash` struct - ideally we would 'const' allocate it, but `String` isn't const compatible. We could maybe use `String::from_raw_parts` but it would depend on too many implementation details. The real solution here is probably to change the definition of PrehashedString to hold an Either or a Cow? not sure if that is worth it. ## Why? Atomic ref counting is cheap but not free, we have a large number of `rcstr!` allocated strings and so it is pure overhead to spend any time refcounting them when the count will never hit zero. See the upstack PR for some performance measurements on vercel-site
1 parent 27e2e74 commit 261a5c3

File tree

3 files changed

+195
-59
lines changed

3 files changed

+195
-59
lines changed

‎turbopack/crates/turbo-rcstr/src/dynamic.rs‎

Lines changed: 76 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -3,12 +3,37 @@ use std::{num::NonZeroU8, ptr::NonNull};
33
use triomphe::Arc;
44

55
use crate::{
6-
INLINE_TAG, INLINE_TAG_INIT, LEN_OFFSET, RcStr, TAG_MASK,
6+
INLINE_TAG, INLINE_TAG_INIT, LEN_OFFSET, RcStr, STATIC_TAG, TAG_MASK,
77
tagged_value::{MAX_INLINE_LEN, TaggedValue},
88
};
99

10-
pub(crate) struct PrehashedString {
11-
pub value: String,
10+
pub enum Payload {
11+
String(String),
12+
Ref(&'static str),
13+
}
14+
15+
impl Payload {
16+
pub(crate) fn as_str(&self) -> &str {
17+
match self {
18+
Payload::String(s) => s,
19+
Payload::Ref(s) => s,
20+
}
21+
}
22+
pub(crate) fn into_string(self) -> String {
23+
match self {
24+
Payload::String(s) => s,
25+
Payload::Ref(r) => r.to_string(),
26+
}
27+
}
28+
}
29+
impl PartialEq for Payload {
30+
fn eq(&self, other: &Self) -> bool {
31+
self.as_str() == other.as_str()
32+
}
33+
}
34+
35+
pub struct PrehashedString {
36+
pub value: Payload,
1237
/// This is not the actual `fxhash`, but rather it's a value that passed to
1338
/// `write_u64` of [rustc_hash::FxHasher].
1439
pub hash: u64,
@@ -46,7 +71,7 @@ pub(crate) fn new_atom<T: AsRef<str> + Into<String>>(text: T) -> RcStr {
4671
let hash = hash_bytes(text.as_ref().as_bytes());
4772

4873
let entry: Arc<PrehashedString> = Arc::new(PrehashedString {
49-
value: text.into(),
74+
value: Payload::String(text.into()),
5075
hash,
5176
});
5277
let entry = Arc::into_raw(entry);
@@ -61,6 +86,22 @@ pub(crate) fn new_atom<T: AsRef<str> + Into<String>>(text: T) -> RcStr {
6186
}
6287
}
6388

89+
#[inline(always)]
90+
pub(crate) fn new_static_atom(string: &'static PrehashedString) -> RcStr {
91+
let mut entry = string as *const PrehashedString;
92+
debug_assert!(0 == entry as u8 & TAG_MASK);
93+
// Tag it as a static pointer
94+
entry = ((entry as usize) | STATIC_TAG as usize) as *mut PrehashedString;
95+
let ptr: NonNull<PrehashedString> = unsafe {
96+
// Safety: Box::into_raw returns a non-null pointer
97+
NonNull::new_unchecked(entry as *mut _)
98+
};
99+
100+
RcStr {
101+
unsafe_data: TaggedValue::new_ptr(ptr),
102+
}
103+
}
104+
64105
/// Attempts to construct an RcStr but only if it can be constructed inline.
65106
/// This is primarily useful in constant contexts.
66107
#[doc(hidden)]
@@ -90,7 +131,7 @@ const SEED2: u64 = 0x13198a2e03707344;
90131
const PREVENT_TRIVIAL_ZERO_COLLAPSE: u64 = 0xa4093822299f31d0;
91132

92133
#[inline]
93-
fn multiply_mix(x: u64, y: u64) -> u64 {
134+
const fn multiply_mix(x: u64, y: u64) -> u64 {
94135
#[cfg(target_pointer_width = "64")]
95136
{
96137
// We compute the full u64 x u64 -> u128 product, this is a single mul
@@ -131,6 +172,26 @@ fn multiply_mix(x: u64, y: u64) -> u64 {
131172
}
132173
}
133174

175+
// Const compatible helper function to read a u64 from a byte array at a given offset
176+
const fn read_u64_le(bytes: &[u8], offset: usize) -> u64 {
177+
(bytes[offset] as u64)
178+
| ((bytes[offset + 1] as u64) << 8)
179+
| ((bytes[offset + 2] as u64) << 16)
180+
| ((bytes[offset + 3] as u64) << 24)
181+
| ((bytes[offset + 4] as u64) << 32)
182+
| ((bytes[offset + 5] as u64) << 40)
183+
| ((bytes[offset + 6] as u64) << 48)
184+
| ((bytes[offset + 7] as u64) << 56)
185+
}
186+
187+
// Const compatible helper function to read a u32 from a byte array at a given offset
188+
const fn read_u32_le(bytes: &[u8], offset: usize) -> u32 {
189+
(bytes[offset] as u32)
190+
| ((bytes[offset + 1] as u32) << 8)
191+
| ((bytes[offset + 2] as u32) << 16)
192+
| ((bytes[offset + 3] as u32) << 24)
193+
}
194+
134195
/// Copied from `hash_bytes` of `rustc-hash`.
135196
///
136197
/// See: https://github.com/rust-lang/rustc-hash/blob/dc5c33f1283de2da64d8d7a06401d91aded03ad4/src/lib.rs#L252-L297
@@ -149,19 +210,20 @@ fn multiply_mix(x: u64, y: u64) -> u64 {
149210
/// We don't bother avalanching here as we'll feed this hash into a
150211
/// multiplication after which we take the high bits, which avalanches for us.
151212
#[inline]
152-
fn hash_bytes(bytes: &[u8]) -> u64 {
213+
#[doc(hidden)]
214+
pub const fn hash_bytes(bytes: &[u8]) -> u64 {
153215
let len = bytes.len();
154216
let mut s0 = SEED1;
155217
let mut s1 = SEED2;
156218

157219
if len <= 16 {
158220
// XOR the input into s0, s1.
159221
if len >= 8 {
160-
s0 ^= u64::from_le_bytes(bytes[0..8].try_into().unwrap());
161-
s1 ^= u64::from_le_bytes(bytes[len - 8..].try_into().unwrap());
222+
s0 ^= read_u64_le(bytes, 0);
223+
s1 ^= read_u64_le(bytes, len - 8);
162224
} else if len >= 4 {
163-
s0 ^= u32::from_le_bytes(bytes[0..4].try_into().unwrap()) as u64;
164-
s1 ^= u32::from_le_bytes(bytes[len - 4..].try_into().unwrap()) as u64;
225+
s0 ^= read_u32_le(bytes, 0) as u64;
226+
s1 ^= read_u32_le(bytes, len - 4) as u64;
165227
} else if len > 0 {
166228
let lo = bytes[0];
167229
let mid = bytes[len / 2];
@@ -173,8 +235,8 @@ fn hash_bytes(bytes: &[u8]) -> u64 {
173235
// Handle bulk (can partially overlap with suffix).
174236
let mut off = 0;
175237
while off < len - 16 {
176-
let x = u64::from_le_bytes(bytes[off..off + 8].try_into().unwrap());
177-
let y = u64::from_le_bytes(bytes[off + 8..off + 16].try_into().unwrap());
238+
let x = read_u64_le(bytes, off);
239+
let y = read_u64_le(bytes, off + 8);
178240

179241
// Replace s1 with a mix of s0, x, and y, and s0 with s1.
180242
// This ensures the compiler can unroll this loop into two
@@ -188,9 +250,8 @@ fn hash_bytes(bytes: &[u8]) -> u64 {
188250
off += 16;
189251
}
190252

191-
let suffix = &bytes[len - 16..];
192-
s0 ^= u64::from_le_bytes(suffix[0..8].try_into().unwrap());
193-
s1 ^= u64::from_le_bytes(suffix[8..16].try_into().unwrap());
253+
s0 ^= read_u64_le(bytes, len - 16);
254+
s1 ^= read_u64_le(bytes, len - 8);
194255
}
195256

196257
multiply_mix(s0, s1) ^ (len as u64)

0 commit comments

Comments
 (0)