From 99ee3dc40273f8a379edd5b41c22513c523ce900 Mon Sep 17 00:00:00 2001 From: Luke Sandberg Date: Wed, 23 Jul 2025 14:57:11 -0700 Subject: [PATCH 1/9] tag `rcstr!` allocated Rcstr values as 'static' and stop refcounting them --- turbopack/crates/turbo-rcstr/src/dynamic.rs | 32 ++++++- turbopack/crates/turbo-rcstr/src/lib.rs | 84 ++++++++++++------- .../crates/turbo-rcstr/src/tagged_value.rs | 17 +++- 3 files changed, 97 insertions(+), 36 deletions(-) diff --git a/turbopack/crates/turbo-rcstr/src/dynamic.rs b/turbopack/crates/turbo-rcstr/src/dynamic.rs index 7e1c34a2e6bb21..08bc970f03d03b 100644 --- a/turbopack/crates/turbo-rcstr/src/dynamic.rs +++ b/turbopack/crates/turbo-rcstr/src/dynamic.rs @@ -3,7 +3,7 @@ use std::{num::NonZeroU8, ptr::NonNull}; use triomphe::Arc; use crate::{ - INLINE_TAG, INLINE_TAG_INIT, LEN_OFFSET, RcStr, TAG_MASK, + INLINE_TAG, INLINE_TAG_INIT, LEN_OFFSET, RcStr, STATIC_TAG, TAG_MASK, tagged_value::{MAX_INLINE_LEN, TaggedValue}, }; @@ -61,6 +61,36 @@ pub(crate) fn new_atom + Into>(text: T) -> RcStr { } } +pub(crate) fn new_static_atom(text: &'static str) -> RcStr { + debug_assert!( + text.len() >= MAX_INLINE_LEN, + "should have use the rcstr! macro? this string is too short" + ); + let hash = hash_bytes(text.as_bytes()); + + let entry: Box = Box::new(PrehashedString { + value: text.into(), + hash, + }); + // Hello memory leak! + // We are leaking here because the caller is asserting that this RcStr will have a static + // lifetime Ideally we wouldn't be copying the static str into the PrehashedString as a + // String but instead keeping the `&'static str` reference somehow, perhaps using a Cow?? or + // perhaps switching to a ThinArc + let mut entry = Box::into_raw(entry); + debug_assert!(0 == entry as u8 & TAG_MASK); + // Tag it as a static pointer + entry = ((entry as usize) | STATIC_TAG as usize) as *mut PrehashedString; + let ptr: NonNull = unsafe { + // Safety: Box::into_raw returns a non-null pointer + NonNull::new_unchecked(entry as *mut _) + }; + + RcStr { + unsafe_data: TaggedValue::new_ptr(ptr), + } +} + /// Attempts to construct an RcStr but only if it can be constructed inline. /// This is primarily useful in constant contexts. #[doc(hidden)] diff --git a/turbopack/crates/turbo-rcstr/src/lib.rs b/turbopack/crates/turbo-rcstr/src/lib.rs index 9839d224c32d38..2e8c175f166eed 100644 --- a/turbopack/crates/turbo-rcstr/src/lib.rs +++ b/turbopack/crates/turbo-rcstr/src/lib.rs @@ -74,32 +74,52 @@ pub struct RcStr { unsafe impl Send for RcStr {} unsafe impl Sync for RcStr {} +// Marks a payload that is stored in an Arc const DYNAMIC_TAG: u8 = 0b_00; +const DYNAMIC_LOCATION: u8 = 0b_0; +// Marks a payload that has been leaked since it has a static lifetime +const STATIC_TAG: u8 = 0b_10; +// The payload is stored inline const INLINE_TAG: u8 = 0b_01; // len in upper nybble +const INLINE_LOCATION: u8 = 0b_1; const INLINE_TAG_INIT: NonZeroU8 = NonZeroU8::new(INLINE_TAG).unwrap(); const TAG_MASK: u8 = 0b_11; +const LOCATION_MASK: u8 = 0b_1; +// For inline tags the length is stored in the upper 4 bits of the tag byte const LEN_OFFSET: usize = 4; const LEN_MASK: u8 = 0xf0; impl RcStr { #[inline(always)] fn tag(&self) -> u8 { - self.unsafe_data.tag() & TAG_MASK + self.unsafe_data.tag_byte() & TAG_MASK + } + #[inline(always)] + fn location(&self) -> u8 { + self.unsafe_data.tag_byte() & LOCATION_MASK } #[inline(never)] pub fn as_str(&self) -> &str { - match self.tag() { - DYNAMIC_TAG => unsafe { dynamic::deref_from(self.unsafe_data).value.as_str() }, - INLINE_TAG => { - let len = (self.unsafe_data.tag() & LEN_MASK) >> LEN_OFFSET; - let src = self.unsafe_data.data(); - unsafe { std::str::from_utf8_unchecked(&src[..(len as usize)]) } - } + match self.location() { + DYNAMIC_LOCATION => self.dynamic_as_str(), + INLINE_LOCATION => self.inline_as_str(), _ => unsafe { debug_unreachable!() }, } } + fn inline_as_str(&self) -> &str { + let len = (self.unsafe_data.tag_byte() & LEN_MASK) >> LEN_OFFSET; + let src = self.unsafe_data.data(); + unsafe { std::str::from_utf8_unchecked(&src[..(len as usize)]) } + } + + // Extract the str reference from a string stored in a dynamic location + fn dynamic_as_str(&self) -> &str { + debug_assert!(self.location() == DYNAMIC_LOCATION); + unsafe { dynamic::deref_from(self.unsafe_data).value.as_str() } + } + /// Returns an owned mutable [`String`]. /// /// This implementation is more efficient than [`ToString::to_string`]: @@ -114,10 +134,11 @@ impl RcStr { let arc = unsafe { dynamic::restore_arc(ManuallyDrop::new(self).unsafe_data) }; match Arc::try_unwrap(arc) { Ok(v) => v.value, - Err(arc) => arc.value.to_string(), + Err(arc) => arc.value.clone(), } } - INLINE_TAG => self.as_str().to_string(), + INLINE_TAG => self.inline_as_str().to_string(), + STATIC_TAG => self.dynamic_as_str().to_string(), _ => unsafe { debug_unreachable!() }, } } @@ -125,19 +146,6 @@ impl RcStr { pub fn map(self, f: impl FnOnce(String) -> String) -> Self { RcStr::from(Cow::Owned(f(self.into_owned()))) } - - #[inline] - pub(crate) fn from_alias(alias: TaggedValue) -> Self { - if alias.tag() & TAG_MASK == DYNAMIC_TAG { - unsafe { - let arc = dynamic::restore_arc(alias); - forget(arc.clone()); - forget(arc); - } - } - - Self { unsafe_data: alias } - } } impl DeterministicHash for RcStr { @@ -264,7 +272,16 @@ impl From for PathBuf { impl Clone for RcStr { #[inline(always)] fn clone(&self) -> Self { - Self::from_alias(self.unsafe_data) + let alias = self.unsafe_data; + if alias.tag_byte() & TAG_MASK == DYNAMIC_TAG { + unsafe { + let arc = dynamic::restore_arc(alias); + forget(arc.clone()); + forget(arc); + } + } + + RcStr { unsafe_data: alias } } } @@ -276,13 +293,13 @@ impl Default for RcStr { impl PartialEq for RcStr { fn eq(&self, other: &Self) -> bool { - match (self.tag(), other.tag()) { - (DYNAMIC_TAG, DYNAMIC_TAG) => { + match (self.location(), other.location()) { + (DYNAMIC_LOCATION, DYNAMIC_LOCATION) => { let l = unsafe { deref_from(self.unsafe_data) }; let r = unsafe { deref_from(other.unsafe_data) }; l.hash == r.hash && l.value == r.value } - (INLINE_TAG, INLINE_TAG) => self.unsafe_data == other.unsafe_data, + (INLINE_LOCATION, INLINE_LOCATION) => self.unsafe_data == other.unsafe_data, _ => false, } } @@ -304,13 +321,13 @@ impl Ord for RcStr { impl Hash for RcStr { fn hash(&self, state: &mut H) { - match self.tag() { - DYNAMIC_TAG => { + match self.location() { + DYNAMIC_LOCATION => { let l = unsafe { deref_from(self.unsafe_data) }; state.write_u64(l.hash); state.write_u8(0xff); } - INLINE_TAG => { + INLINE_LOCATION => { self.as_str().hash(state); } _ => unsafe { debug_unreachable!() }, @@ -344,6 +361,11 @@ pub const fn inline_atom(s: &str) -> Option { dynamic::inline_atom(s) } +#[doc(hidden)] +pub fn from_static(s: &'static str) -> RcStr { + dynamic::new_static_atom(s) +} + /// Create an rcstr from a string literal. /// allocates the RcStr inline when possible otherwise uses a `LazyLock` to manage the allocation. #[macro_export] @@ -357,7 +379,7 @@ macro_rules! rcstr { #[inline(never)] fn get_rcstr() -> $crate::RcStr { static CACHE: std::sync::LazyLock<$crate::RcStr> = - std::sync::LazyLock::new(|| $crate::RcStr::from($s)); + std::sync::LazyLock::new(|| $crate::from_static($s)); (*CACHE).clone() } diff --git a/turbopack/crates/turbo-rcstr/src/tagged_value.rs b/turbopack/crates/turbo-rcstr/src/tagged_value.rs index aa75def5544693..d1329535aea69f 100644 --- a/turbopack/crates/turbo-rcstr/src/tagged_value.rs +++ b/turbopack/crates/turbo-rcstr/src/tagged_value.rs @@ -3,6 +3,13 @@ use std::{num::NonZeroU8, os::raw::c_void, ptr::NonNull, slice}; use self::raw_types::*; +#[cfg(not(any( + target_pointer_width = "32", + target_pointer_width = "16", + feature = "atom_size_64", + feature = "atom_size_128" +)))] +use crate::TAG_MASK; #[cfg(feature = "atom_size_128")] mod raw_types { @@ -89,7 +96,9 @@ impl TaggedValue { feature = "atom_size_128" ))] { - self.value.get() as usize as _ + use crate::TAG_MASK; + + (self.value.get() as usize & !(TAG_MASK as usize)) as _ } #[cfg(not(any( target_pointer_width = "32", @@ -97,8 +106,8 @@ impl TaggedValue { feature = "atom_size_64", feature = "atom_size_128" )))] - unsafe { - std::mem::transmute(Some(self.value)) + { + (self.value.as_ptr() as usize & !(TAG_MASK as usize)) as _ } } @@ -108,7 +117,7 @@ impl TaggedValue { } #[inline(always)] - pub fn tag(&self) -> u8 { + pub fn tag_byte(&self) -> u8 { (self.get_value() & 0xff) as u8 } From e13c7429eff3dbaa391fe9f053e296e2a52d489b Mon Sep 17 00:00:00 2001 From: Luke Sandberg Date: Wed, 23 Jul 2025 16:43:58 -0700 Subject: [PATCH 2/9] update our equal impl and add a test --- turbopack/crates/turbo-rcstr/src/lib.rs | 26 ++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/turbopack/crates/turbo-rcstr/src/lib.rs b/turbopack/crates/turbo-rcstr/src/lib.rs index 2e8c175f166eed..1c9386af8f0588 100644 --- a/turbopack/crates/turbo-rcstr/src/lib.rs +++ b/turbopack/crates/turbo-rcstr/src/lib.rs @@ -71,6 +71,11 @@ pub struct RcStr { unsafe_data: TaggedValue, } +const _: () = { + // Enforce that RcStr triggers the non-zero size optimization. + assert!(std::mem::size_of::() == std::mem::size_of::>()); +}; + unsafe impl Send for RcStr {} unsafe impl Sync for RcStr {} @@ -293,13 +298,18 @@ impl Default for RcStr { impl PartialEq for RcStr { fn eq(&self, other: &Self) -> bool { + // For inline RcStrs this is sufficient and for out of line values it handles a simple + // identity cases + if self.unsafe_data == self.unsafe_data { + return true; + } + // They can still be equal if they are both stored on the heap match (self.location(), other.location()) { (DYNAMIC_LOCATION, DYNAMIC_LOCATION) => { let l = unsafe { deref_from(self.unsafe_data) }; let r = unsafe { deref_from(other.unsafe_data) }; l.hash == r.hash && l.value == r.value } - (INLINE_LOCATION, INLINE_LOCATION) => self.unsafe_data == other.unsafe_data, _ => false, } } @@ -350,8 +360,9 @@ impl<'de> Deserialize<'de> for RcStr { impl Drop for RcStr { fn drop(&mut self) { - if self.tag() == DYNAMIC_TAG { - unsafe { drop(dynamic::restore_arc(self.unsafe_data)) } + let alias = self.unsafe_data; + if alias.tag_byte() & TAG_MASK == DYNAMIC_TAG { + unsafe { drop(dynamic::restore_arc(alias)) } } } } @@ -482,6 +493,15 @@ mod tests { assert_eq!(rcstr!("abcdefghi"), RcStr::from("abcdefghi")); } + #[test] + fn test_static_atom() { + const LONG: &str = "a very long string that lives forever"; + let leaked = rcstr!(LONG); + let not_leaked = RcStr::from(LONG); + assert_ne!(leaked.tag(), not_leaked.tag()); + assert_eq!(leaked, not_leaked); + } + #[test] fn test_inline_atom() { // This is a silly test, just asserts that we can evaluate this in a constant context. From 68d720ede54b134edd4d01d8bd110b4d16fe4d85 Mon Sep 17 00:00:00 2001 From: Luke Sandberg Date: Thu, 24 Jul 2025 12:54:21 -0700 Subject: [PATCH 3/9] Precompute static hashes, use static allocation instead of dynamic optimize clones out of the static cache --- turbopack/crates/turbo-rcstr/src/dynamic.rs | 64 +++++++++++---------- turbopack/crates/turbo-rcstr/src/lib.rs | 36 +++++++++--- 2 files changed, 64 insertions(+), 36 deletions(-) diff --git a/turbopack/crates/turbo-rcstr/src/dynamic.rs b/turbopack/crates/turbo-rcstr/src/dynamic.rs index 08bc970f03d03b..3aff3d7163e482 100644 --- a/turbopack/crates/turbo-rcstr/src/dynamic.rs +++ b/turbopack/crates/turbo-rcstr/src/dynamic.rs @@ -7,7 +7,7 @@ use crate::{ tagged_value::{MAX_INLINE_LEN, TaggedValue}, }; -pub(crate) struct PrehashedString { +pub struct PrehashedString { pub value: String, /// This is not the actual `fxhash`, but rather it's a value that passed to /// `write_u64` of [rustc_hash::FxHasher]. @@ -61,23 +61,9 @@ pub(crate) fn new_atom + Into>(text: T) -> RcStr { } } -pub(crate) fn new_static_atom(text: &'static str) -> RcStr { - debug_assert!( - text.len() >= MAX_INLINE_LEN, - "should have use the rcstr! macro? this string is too short" - ); - let hash = hash_bytes(text.as_bytes()); - - let entry: Box = Box::new(PrehashedString { - value: text.into(), - hash, - }); - // Hello memory leak! - // We are leaking here because the caller is asserting that this RcStr will have a static - // lifetime Ideally we wouldn't be copying the static str into the PrehashedString as a - // String but instead keeping the `&'static str` reference somehow, perhaps using a Cow?? or - // perhaps switching to a ThinArc - let mut entry = Box::into_raw(entry); +#[inline(always)] +pub(crate) fn new_static_atom(string: &'static PrehashedString) -> RcStr { + let mut entry = string as *const PrehashedString; debug_assert!(0 == entry as u8 & TAG_MASK); // Tag it as a static pointer entry = ((entry as usize) | STATIC_TAG as usize) as *mut PrehashedString; @@ -120,7 +106,7 @@ const SEED2: u64 = 0x13198a2e03707344; const PREVENT_TRIVIAL_ZERO_COLLAPSE: u64 = 0xa4093822299f31d0; #[inline] -fn multiply_mix(x: u64, y: u64) -> u64 { +const fn multiply_mix(x: u64, y: u64) -> u64 { #[cfg(target_pointer_width = "64")] { // We compute the full u64 x u64 -> u128 product, this is a single mul @@ -161,6 +147,26 @@ fn multiply_mix(x: u64, y: u64) -> u64 { } } +// Const compatible helper function to read a u64 from a byte array at a given offset +const fn read_u64_le(bytes: &[u8], offset: usize) -> u64 { + (bytes[offset] as u64) + | ((bytes[offset + 1] as u64) << 8) + | ((bytes[offset + 2] as u64) << 16) + | ((bytes[offset + 3] as u64) << 24) + | ((bytes[offset + 4] as u64) << 32) + | ((bytes[offset + 5] as u64) << 40) + | ((bytes[offset + 6] as u64) << 48) + | ((bytes[offset + 7] as u64) << 56) +} + +// Const compatible helper function to read a u32 from a byte array at a given offset +const fn read_u32_le(bytes: &[u8], offset: usize) -> u32 { + (bytes[offset] as u32) + | ((bytes[offset + 1] as u32) << 8) + | ((bytes[offset + 2] as u32) << 16) + | ((bytes[offset + 3] as u32) << 24) +} + /// Copied from `hash_bytes` of `rustc-hash`. /// /// See: https://github.com/rust-lang/rustc-hash/blob/dc5c33f1283de2da64d8d7a06401d91aded03ad4/src/lib.rs#L252-L297 @@ -179,7 +185,8 @@ fn multiply_mix(x: u64, y: u64) -> u64 { /// We don't bother avalanching here as we'll feed this hash into a /// multiplication after which we take the high bits, which avalanches for us. #[inline] -fn hash_bytes(bytes: &[u8]) -> u64 { +#[doc(hidden)] +pub const fn hash_bytes(bytes: &[u8]) -> u64 { let len = bytes.len(); let mut s0 = SEED1; let mut s1 = SEED2; @@ -187,11 +194,11 @@ fn hash_bytes(bytes: &[u8]) -> u64 { if len <= 16 { // XOR the input into s0, s1. if len >= 8 { - s0 ^= u64::from_le_bytes(bytes[0..8].try_into().unwrap()); - s1 ^= u64::from_le_bytes(bytes[len - 8..].try_into().unwrap()); + s0 ^= read_u64_le(bytes, 0); + s1 ^= read_u64_le(bytes, len - 8); } else if len >= 4 { - s0 ^= u32::from_le_bytes(bytes[0..4].try_into().unwrap()) as u64; - s1 ^= u32::from_le_bytes(bytes[len - 4..].try_into().unwrap()) as u64; + s0 ^= read_u32_le(bytes, 0) as u64; + s1 ^= read_u32_le(bytes, len - 4) as u64; } else if len > 0 { let lo = bytes[0]; let mid = bytes[len / 2]; @@ -203,8 +210,8 @@ fn hash_bytes(bytes: &[u8]) -> u64 { // Handle bulk (can partially overlap with suffix). let mut off = 0; while off < len - 16 { - let x = u64::from_le_bytes(bytes[off..off + 8].try_into().unwrap()); - let y = u64::from_le_bytes(bytes[off + 8..off + 16].try_into().unwrap()); + let x = read_u64_le(bytes, off); + let y = read_u64_le(bytes, off + 8); // Replace s1 with a mix of s0, x, and y, and s0 with s1. // This ensures the compiler can unroll this loop into two @@ -218,9 +225,8 @@ fn hash_bytes(bytes: &[u8]) -> u64 { off += 16; } - let suffix = &bytes[len - 16..]; - s0 ^= u64::from_le_bytes(suffix[0..8].try_into().unwrap()); - s1 ^= u64::from_le_bytes(suffix[8..16].try_into().unwrap()); + s0 ^= read_u64_le(bytes, len - 16); + s1 ^= read_u64_le(bytes, len - 8); } multiply_mix(s0, s1) ^ (len as u64) diff --git a/turbopack/crates/turbo-rcstr/src/lib.rs b/turbopack/crates/turbo-rcstr/src/lib.rs index 1c9386af8f0588..40caa1fae06bdf 100644 --- a/turbopack/crates/turbo-rcstr/src/lib.rs +++ b/turbopack/crates/turbo-rcstr/src/lib.rs @@ -300,7 +300,7 @@ impl PartialEq for RcStr { fn eq(&self, other: &Self) -> bool { // For inline RcStrs this is sufficient and for out of line values it handles a simple // identity cases - if self.unsafe_data == self.unsafe_data { + if self.unsafe_data == other.unsafe_data { return true; } // They can still be equal if they are both stored on the heap @@ -367,15 +367,30 @@ impl Drop for RcStr { } } +// Exports for our macro #[doc(hidden)] pub const fn inline_atom(s: &str) -> Option { dynamic::inline_atom(s) } #[doc(hidden)] -pub fn from_static(s: &'static str) -> RcStr { +pub fn from_static(s: &'static PrehashedString) -> RcStr { dynamic::new_static_atom(s) } +#[doc(hidden)] +pub use dynamic::{PrehashedString, hash_bytes}; + +#[doc(hidden)] +impl RcStr { + // Allow the rcstr! macro to skip a tag branch + #[doc(hidden)] + pub fn unsafe_copy_for_macro(&self) -> RcStr { + debug_assert!(self.tag() == STATIC_TAG); + Self { + unsafe_data: self.unsafe_data, + } + } +} /// Create an rcstr from a string literal. /// allocates the RcStr inline when possible otherwise uses a `LazyLock` to manage the allocation. @@ -383,16 +398,23 @@ pub fn from_static(s: &'static str) -> RcStr { macro_rules! rcstr { ($s:expr) => {{ const INLINE: core::option::Option<$crate::RcStr> = $crate::inline_atom($s); - // this condition should be able to be compile time evaluated and inlined. + // This condition can be compile time evaluated and inlined. if INLINE.is_some() { INLINE.unwrap() } else { #[inline(never)] fn get_rcstr() -> $crate::RcStr { - static CACHE: std::sync::LazyLock<$crate::RcStr> = - std::sync::LazyLock::new(|| $crate::from_static($s)); - - (*CACHE).clone() + // Allocate static storage for the PrehashedString + static INNER: ::std::sync::LazyLock<$crate::PrehashedString> = + ::std::sync::LazyLock::new(|| $crate::PrehashedString { + value: $s.into(), + // compute the hash at compile time + hash: const { $crate::hash_bytes($s.as_bytes()) }, + }); + static CACHE: ::std::sync::LazyLock<$crate::RcStr> = + ::std::sync::LazyLock::new(|| $crate::from_static(&*INNER)); + + (*CACHE).unsafe_copy_for_macro() } get_rcstr() } From cfebbf30436256a99a6aca6b5cd52b3267eab6f4 Mon Sep 17 00:00:00 2001 From: Luke Sandberg Date: Thu, 24 Jul 2025 13:19:56 -0700 Subject: [PATCH 4/9] comment --- turbopack/crates/turbo-rcstr/src/lib.rs | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/turbopack/crates/turbo-rcstr/src/lib.rs b/turbopack/crates/turbo-rcstr/src/lib.rs index 40caa1fae06bdf..4c35ef2af06e2c 100644 --- a/turbopack/crates/turbo-rcstr/src/lib.rs +++ b/turbopack/crates/turbo-rcstr/src/lib.rs @@ -1,4 +1,5 @@ use std::{ + alloc::GlobalAlloc, borrow::{Borrow, Cow}, ffi::OsStr, fmt::{Debug, Display}, @@ -11,6 +12,7 @@ use std::{ use bytes_str::BytesStr; use debug_unreachable::debug_unreachable; +use napi::threadsafe_function::ErrorStrategy::T; use serde::{Deserialize, Deserializer, Serialize, Serializer}; use shrink_to_fit::ShrinkToFit; use triomphe::Arc; @@ -382,7 +384,7 @@ pub use dynamic::{PrehashedString, hash_bytes}; #[doc(hidden)] impl RcStr { - // Allow the rcstr! macro to skip a tag branch + // Allow the rcstr! macro to skip a tag branch and just copy the struct #[doc(hidden)] pub fn unsafe_copy_for_macro(&self) -> RcStr { debug_assert!(self.tag() == STATIC_TAG); @@ -407,6 +409,14 @@ macro_rules! rcstr { // Allocate static storage for the PrehashedString static INNER: ::std::sync::LazyLock<$crate::PrehashedString> = ::std::sync::LazyLock::new(|| $crate::PrehashedString { + // `String::from_raw_parts`` would be a pretty legit option here since + // we could const allocate this. However, we would need to: + // * cast the static str to a *mut u8 + // * convince ourselves that the admonitions to supply allocator allocated + // pointers are only about drop. + // Alternatively we could transmute a Vec allocated from from_raw_parts_in + // using an allocator that never frees. + // For how we just copy and allocate value: $s.into(), // compute the hash at compile time hash: const { $crate::hash_bytes($s.as_bytes()) }, From 165ee922ac012d3118b765e317a4e276f12abc9c Mon Sep 17 00:00:00 2001 From: Luke Sandberg Date: Thu, 24 Jul 2025 13:38:34 -0700 Subject: [PATCH 5/9] out out damn spot --- turbopack/crates/turbo-rcstr/src/lib.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/turbopack/crates/turbo-rcstr/src/lib.rs b/turbopack/crates/turbo-rcstr/src/lib.rs index 4c35ef2af06e2c..f2f82f0e894768 100644 --- a/turbopack/crates/turbo-rcstr/src/lib.rs +++ b/turbopack/crates/turbo-rcstr/src/lib.rs @@ -1,5 +1,4 @@ use std::{ - alloc::GlobalAlloc, borrow::{Borrow, Cow}, ffi::OsStr, fmt::{Debug, Display}, @@ -12,7 +11,6 @@ use std::{ use bytes_str::BytesStr; use debug_unreachable::debug_unreachable; -use napi::threadsafe_function::ErrorStrategy::T; use serde::{Deserialize, Deserializer, Serialize, Serializer}; use shrink_to_fit::ShrinkToFit; use triomphe::Arc; From 71720e4fcba155b64bce361e51d7c938b999b6f3 Mon Sep 17 00:00:00 2001 From: Luke Sandberg Date: Fri, 25 Jul 2025 08:32:55 -0700 Subject: [PATCH 6/9] review feedback --- turbopack/crates/turbo-rcstr/src/lib.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/turbopack/crates/turbo-rcstr/src/lib.rs b/turbopack/crates/turbo-rcstr/src/lib.rs index f2f82f0e894768..e93b7985b66a80 100644 --- a/turbopack/crates/turbo-rcstr/src/lib.rs +++ b/turbopack/crates/turbo-rcstr/src/lib.rs @@ -360,9 +360,8 @@ impl<'de> Deserialize<'de> for RcStr { impl Drop for RcStr { fn drop(&mut self) { - let alias = self.unsafe_data; - if alias.tag_byte() & TAG_MASK == DYNAMIC_TAG { - unsafe { drop(dynamic::restore_arc(alias)) } + if self.tag() == DYNAMIC_TAG { + unsafe { drop(dynamic::restore_arc(self.unsafe_data)) } } } } From b0f61cbeb2379b0e30bd097f9e0d2b6cf2cf5dbc Mon Sep 17 00:00:00 2001 From: Luke Sandberg Date: Fri, 25 Jul 2025 10:33:59 -0700 Subject: [PATCH 7/9] Change the `rcstr!` macro to avoid lazy locks This removes branches from the allocation path and instead we just need to tag the pointer on each call. To do this we needed PrehashedString to be const compatible which required some indirection to allow it to just directly store static strings. This will add a bit of overhead to `as_str`, `eq` and `drop` but speed up allocation. --- turbopack/crates/turbo-rcstr/src/dynamic.rs | 29 ++++++++++++- turbopack/crates/turbo-rcstr/src/lib.rs | 45 +++++++-------------- 2 files changed, 41 insertions(+), 33 deletions(-) diff --git a/turbopack/crates/turbo-rcstr/src/dynamic.rs b/turbopack/crates/turbo-rcstr/src/dynamic.rs index 3aff3d7163e482..31b96beee01da8 100644 --- a/turbopack/crates/turbo-rcstr/src/dynamic.rs +++ b/turbopack/crates/turbo-rcstr/src/dynamic.rs @@ -7,8 +7,33 @@ use crate::{ tagged_value::{MAX_INLINE_LEN, TaggedValue}, }; +pub enum Payload { + String(String), + Ref(&'static str), +} + +impl Payload { + pub(crate) fn as_str(&self) -> &str { + match self { + Payload::String(s) => s, + Payload::Ref(s) => s, + } + } + pub(crate) fn into_string(self) -> String { + match self { + Payload::String(s) => s, + Payload::Ref(r) => r.to_string(), + } + } +} +impl PartialEq for Payload { + fn eq(&self, other: &Self) -> bool { + self.as_str() == other.as_str() + } +} + pub struct PrehashedString { - pub value: String, + pub value: Payload, /// This is not the actual `fxhash`, but rather it's a value that passed to /// `write_u64` of [rustc_hash::FxHasher]. pub hash: u64, @@ -46,7 +71,7 @@ pub(crate) fn new_atom + Into>(text: T) -> RcStr { let hash = hash_bytes(text.as_ref().as_bytes()); let entry: Arc = Arc::new(PrehashedString { - value: text.into(), + value: Payload::String(text.into()), hash, }); let entry = Arc::into_raw(entry); diff --git a/turbopack/crates/turbo-rcstr/src/lib.rs b/turbopack/crates/turbo-rcstr/src/lib.rs index e93b7985b66a80..799409bf6358cb 100644 --- a/turbopack/crates/turbo-rcstr/src/lib.rs +++ b/turbopack/crates/turbo-rcstr/src/lib.rs @@ -17,7 +17,7 @@ use triomphe::Arc; use turbo_tasks_hash::{DeterministicHash, DeterministicHasher}; use crate::{ - dynamic::{deref_from, new_atom}, + dynamic::{deref_from, hash_bytes, new_atom}, tagged_value::TaggedValue, }; @@ -138,8 +138,8 @@ impl RcStr { // convert `self` into `arc` let arc = unsafe { dynamic::restore_arc(ManuallyDrop::new(self).unsafe_data) }; match Arc::try_unwrap(arc) { - Ok(v) => v.value, - Err(arc) => arc.value.clone(), + Ok(v) => v.value.into_string(), + Err(arc) => arc.value.as_str().to_string(), } } INLINE_TAG => self.inline_as_str().to_string(), @@ -373,21 +373,18 @@ pub const fn inline_atom(s: &str) -> Option { } #[doc(hidden)] +#[inline(always)] pub fn from_static(s: &'static PrehashedString) -> RcStr { dynamic::new_static_atom(s) } #[doc(hidden)] -pub use dynamic::{PrehashedString, hash_bytes}; +pub use dynamic::PrehashedString; #[doc(hidden)] -impl RcStr { - // Allow the rcstr! macro to skip a tag branch and just copy the struct - #[doc(hidden)] - pub fn unsafe_copy_for_macro(&self) -> RcStr { - debug_assert!(self.tag() == STATIC_TAG); - Self { - unsafe_data: self.unsafe_data, - } +pub const fn make_const_prehashed_string(text: &'static str) -> PrehashedString { + PrehashedString { + value: dynamic::Payload::Ref(text), + hash: hash_bytes(text.as_bytes()), } } @@ -401,27 +398,13 @@ macro_rules! rcstr { if INLINE.is_some() { INLINE.unwrap() } else { - #[inline(never)] fn get_rcstr() -> $crate::RcStr { // Allocate static storage for the PrehashedString - static INNER: ::std::sync::LazyLock<$crate::PrehashedString> = - ::std::sync::LazyLock::new(|| $crate::PrehashedString { - // `String::from_raw_parts`` would be a pretty legit option here since - // we could const allocate this. However, we would need to: - // * cast the static str to a *mut u8 - // * convince ourselves that the admonitions to supply allocator allocated - // pointers are only about drop. - // Alternatively we could transmute a Vec allocated from from_raw_parts_in - // using an allocator that never frees. - // For how we just copy and allocate - value: $s.into(), - // compute the hash at compile time - hash: const { $crate::hash_bytes($s.as_bytes()) }, - }); - static CACHE: ::std::sync::LazyLock<$crate::RcStr> = - ::std::sync::LazyLock::new(|| $crate::from_static(&*INNER)); - - (*CACHE).unsafe_copy_for_macro() + static RCSTR_STORAGE: $crate::PrehashedString = + $crate::make_const_prehashed_string($s); + // This basically just tags a bit onto the raw pointer and wraps it in an RcStr + // should be fast enough to do every time. + $crate::from_static(&RCSTR_STORAGE) } get_rcstr() } From 6353683022bf368089fac188d261e194a3405750 Mon Sep 17 00:00:00 2001 From: Luke Sandberg Date: Fri, 25 Jul 2025 10:42:34 -0700 Subject: [PATCH 8/9] leave a comment to get vade to understand what is happening --- turbopack/crates/turbo-rcstr/src/lib.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/turbopack/crates/turbo-rcstr/src/lib.rs b/turbopack/crates/turbo-rcstr/src/lib.rs index 799409bf6358cb..faa84a79c8a83a 100644 --- a/turbopack/crates/turbo-rcstr/src/lib.rs +++ b/turbopack/crates/turbo-rcstr/src/lib.rs @@ -310,6 +310,8 @@ impl PartialEq for RcStr { let r = unsafe { deref_from(other.unsafe_data) }; l.hash == r.hash && l.value == r.value } + // NOTE: it is never possible for an inline storage string to compare equal to a dynamic + // allocated string, the construction routines separate the strings based on length. _ => false, } } From 5493b7bbfa7798c62b7416dca4ca298fcdf0291c Mon Sep 17 00:00:00 2001 From: Luke Sandberg Date: Fri, 25 Jul 2025 11:29:07 -0700 Subject: [PATCH 9/9] make bots happier --- turbopack/crates/turbo-rcstr/src/lib.rs | 30 ++++++++++++++++--------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/turbopack/crates/turbo-rcstr/src/lib.rs b/turbopack/crates/turbo-rcstr/src/lib.rs index faa84a79c8a83a..b43c61fbf2593a 100644 --- a/turbopack/crates/turbo-rcstr/src/lib.rs +++ b/turbopack/crates/turbo-rcstr/src/lib.rs @@ -81,7 +81,7 @@ unsafe impl Sync for RcStr {} // Marks a payload that is stored in an Arc const DYNAMIC_TAG: u8 = 0b_00; -const DYNAMIC_LOCATION: u8 = 0b_0; +const PREHASHED_STRING_LOCATION: u8 = 0b_0; // Marks a payload that has been leaked since it has a static lifetime const STATIC_TAG: u8 = 0b_10; // The payload is stored inline @@ -107,21 +107,22 @@ impl RcStr { #[inline(never)] pub fn as_str(&self) -> &str { match self.location() { - DYNAMIC_LOCATION => self.dynamic_as_str(), + PREHASHED_STRING_LOCATION => self.prehashed_string_as_str(), INLINE_LOCATION => self.inline_as_str(), _ => unsafe { debug_unreachable!() }, } } fn inline_as_str(&self) -> &str { + debug_assert!(self.location() == INLINE_LOCATION); let len = (self.unsafe_data.tag_byte() & LEN_MASK) >> LEN_OFFSET; let src = self.unsafe_data.data(); unsafe { std::str::from_utf8_unchecked(&src[..(len as usize)]) } } - // Extract the str reference from a string stored in a dynamic location - fn dynamic_as_str(&self) -> &str { - debug_assert!(self.location() == DYNAMIC_LOCATION); + // Extract the str reference from a string stored in a PrehashedString + fn prehashed_string_as_str(&self) -> &str { + debug_assert!(self.location() == PREHASHED_STRING_LOCATION); unsafe { dynamic::deref_from(self.unsafe_data).value.as_str() } } @@ -143,7 +144,7 @@ impl RcStr { } } INLINE_TAG => self.inline_as_str().to_string(), - STATIC_TAG => self.dynamic_as_str().to_string(), + STATIC_TAG => self.prehashed_string_as_str().to_string(), _ => unsafe { debug_unreachable!() }, } } @@ -278,6 +279,8 @@ impl Clone for RcStr { #[inline(always)] fn clone(&self) -> Self { let alias = self.unsafe_data; + // We only need to increment the ref count for DYNAMIC_TAG values + // For STATIC_TAG and INLINE_TAG we can just copy the value. if alias.tag_byte() & TAG_MASK == DYNAMIC_TAG { unsafe { let arc = dynamic::restore_arc(alias); @@ -305,7 +308,7 @@ impl PartialEq for RcStr { } // They can still be equal if they are both stored on the heap match (self.location(), other.location()) { - (DYNAMIC_LOCATION, DYNAMIC_LOCATION) => { + (PREHASHED_STRING_LOCATION, PREHASHED_STRING_LOCATION) => { let l = unsafe { deref_from(self.unsafe_data) }; let r = unsafe { deref_from(other.unsafe_data) }; l.hash == r.hash && l.value == r.value @@ -334,7 +337,7 @@ impl Ord for RcStr { impl Hash for RcStr { fn hash(&self, state: &mut H) { match self.location() { - DYNAMIC_LOCATION => { + PREHASHED_STRING_LOCATION => { let l = unsafe { deref_from(self.unsafe_data) }; state.write_u64(l.hash); state.write_u8(0xff); @@ -362,8 +365,15 @@ impl<'de> Deserialize<'de> for RcStr { impl Drop for RcStr { fn drop(&mut self) { - if self.tag() == DYNAMIC_TAG { - unsafe { drop(dynamic::restore_arc(self.unsafe_data)) } + match self.tag() { + DYNAMIC_TAG => unsafe { drop(dynamic::restore_arc(self.unsafe_data)) }, + STATIC_TAG => { + // do nothing, these are never deallocated + } + INLINE_TAG => { + // do nothing, these payloads need no drop logic + } + _ => unsafe { debug_unreachable!() }, } } }