Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit ace4574

Browse files
sokrasokra
authored
Turbopack: compute ordered entries in module batches (#77294)
### What? When computing the module batches, also compute the correct postorder of isolated and shared chunk groups
1 parent f67f44c commit ace4574

File tree

3 files changed

+172
-19
lines changed

3 files changed

+172
-19
lines changed

‎turbopack/crates/turbopack-core/src/chunk/mod.rs‎

Lines changed: 25 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -205,25 +205,34 @@ pub enum ChunkingType {
205205

206206
impl ChunkingType {
207207
pub fn is_inherit_async(&self) -> bool {
208-
match self {
209-
ChunkingType::Parallel => false,
210-
ChunkingType::ParallelInheritAsync => true,
211-
ChunkingType::Async => false,
212-
ChunkingType::Isolated { .. } => false,
213-
ChunkingType::Shared { inherit_async, .. } => *inherit_async,
214-
ChunkingType::Traced => false,
215-
}
208+
matches!(
209+
self,
210+
ChunkingType::ParallelInheritAsync
211+
| ChunkingType::Shared {
212+
inherit_async: true,
213+
..
214+
}
215+
)
216216
}
217217

218218
pub fn is_parallel(&self) -> bool {
219-
match self {
220-
ChunkingType::Parallel => true,
221-
ChunkingType::ParallelInheritAsync => true,
222-
ChunkingType::Async => false,
223-
ChunkingType::Isolated { .. } => false,
224-
ChunkingType::Shared { .. } => false,
225-
ChunkingType::Traced => false,
226-
}
219+
matches!(
220+
self,
221+
ChunkingType::Parallel | ChunkingType::ParallelInheritAsync
222+
)
223+
}
224+
225+
pub fn is_merged(&self) -> bool {
226+
matches!(
227+
self,
228+
ChunkingType::Isolated {
229+
merge_tag: Some(_),
230+
..
231+
} | ChunkingType::Shared {
232+
merge_tag: Some(_),
233+
..
234+
}
235+
)
227236
}
228237

229238
pub fn without_inherit_async(&self) -> Self {

‎turbopack/crates/turbopack-core/src/module_graph/chunk_group_info.rs‎

Lines changed: 33 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,8 @@ pub struct ChunkGroupInfo {
8888
pub module_chunk_groups: FxHashMap<ResolvedVc<Box<dyn Module>>, RoaringBitmapWrapper>,
8989
#[turbo_tasks(trace_ignore)]
9090
pub chunk_groups: FxIndexSet<ChunkGroup>,
91+
#[turbo_tasks(trace_ignore)]
92+
pub chunk_group_keys: FxIndexSet<ChunkGroupKey>,
9193
}
9294

9395
#[turbo_tasks::value_impl]
@@ -174,6 +176,19 @@ pub enum ChunkGroup {
174176
}
175177

176178
impl ChunkGroup {
179+
/// Returns the parent group when this chunk group is a merged group. In that case `entries()`
180+
/// are in unspecified order.
181+
pub fn get_merged_parent(&self) -> Option<usize> {
182+
match self {
183+
ChunkGroup::IsolatedMerged { parent, .. } | ChunkGroup::SharedMerged { parent, .. } => {
184+
Some(*parent)
185+
}
186+
_ => None,
187+
}
188+
}
189+
190+
/// Iterates over the entries of the chunk group. When `get_merged_parent` is Some, the order is
191+
/// unspecified.
177192
pub fn entries(&self) -> impl Iterator<Item = ResolvedVc<Box<dyn Module>>> + Clone + '_ {
178193
match self {
179194
ChunkGroup::Async(e) | ChunkGroup::Isolated(e) | ChunkGroup::Shared(e) => {
@@ -185,6 +200,15 @@ impl ChunkGroup {
185200
}
186201
}
187202

203+
pub fn entries_count(&self) -> usize {
204+
match self {
205+
ChunkGroup::Async(_) | ChunkGroup::Isolated(_) | ChunkGroup::Shared(_) => 1,
206+
ChunkGroup::Entry(entries)
207+
| ChunkGroup::IsolatedMerged { entries, .. }
208+
| ChunkGroup::SharedMerged { entries, .. } => entries.len(),
209+
}
210+
}
211+
188212
pub async fn debug_str(&self, chunk_group_info: &ChunkGroupInfo) -> Result<String> {
189213
Ok(match self {
190214
ChunkGroup::Entry(entries) => format!(
@@ -245,8 +269,8 @@ impl ChunkGroup {
245269
}
246270
}
247271

248-
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
249-
enum ChunkGroupKey {
272+
#[derive(Debug, Clone, Hash, PartialEq, Eq, Serialize, Deserialize)]
273+
pub enum ChunkGroupKey {
250274
/// e.g. a page
251275
Entry(Vec<ResolvedVc<Box<dyn Module>>>),
252276
/// a module with an incoming async edge
@@ -270,6 +294,12 @@ enum ChunkGroupKey {
270294
#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
271295
pub struct ChunkGroupId(u32);
272296

297+
impl From<usize> for ChunkGroupId {
298+
fn from(id: usize) -> Self {
299+
Self(id as u32)
300+
}
301+
}
302+
273303
impl Deref for ChunkGroupId {
274304
type Target = u32;
275305
fn deref(&self) -> &Self::Target {
@@ -621,6 +651,7 @@ pub async fn compute_chunk_group_info(graph: &ModuleGraph) -> Result<Vc<ChunkGro
621651

622652
Ok(ChunkGroupInfo {
623653
module_chunk_groups,
654+
chunk_group_keys: chunk_groups_map.keys().cloned().collect(),
624655
chunk_groups: chunk_groups_map
625656
.into_iter()
626657
.map(|(k, (_, merged_entries))| match k {

‎turbopack/crates/turbopack-core/src/module_graph/module_batches.rs‎

Lines changed: 114 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ use crate::{
2020
chunk::{ChunkableModule, ChunkingType},
2121
module::Module,
2222
module_graph::{
23-
chunk_group_info::{ChunkGroupInfo, RoaringBitmapWrapper},
23+
chunk_group_info::{ChunkGroupInfo, ChunkGroupKey, RoaringBitmapWrapper},
2424
module_batch::{ModuleBatch, ModuleBatchGroup, ModuleOrBatch},
2525
traced_di_graph::{iter_neighbors_rev, TracedDiGraph},
2626
GraphTraversalAction, ModuleGraph,
@@ -48,6 +48,8 @@ pub struct ModuleBatchesGraphEdge {
4848
pub module: Option<ResolvedVc<Box<dyn Module>>>,
4949
}
5050

51+
type EntriesList = FxIndexSet<ResolvedVc<Box<dyn Module>>>;
52+
5153
#[turbo_tasks::value(cell = "new", eq = "manual", into = "new")]
5254
pub struct ModuleBatchesGraph {
5355
graph: TracedDiGraph<ModuleOrBatch, ModuleBatchesGraphEdge>,
@@ -61,6 +63,12 @@ pub struct ModuleBatchesGraph {
6163
#[turbo_tasks(trace_ignore)]
6264
entries: FxHashMap<ResolvedVc<Box<dyn Module>>, NodeIndex>,
6365
batch_groups: FxHashMap<ModuleOrBatch, ResolvedVc<ModuleBatchGroup>>,
66+
67+
/// For chunk groups where the postorder of entries is different than the order of the
68+
/// `ChunkGroup::entries()` this contains Some with the postorder list of entries of that chunk
69+
/// group. The index in this list corresponds to the index in the
70+
/// chunk_group_info.chunk_groups.
71+
ordered_entries: Vec<Option<EntriesList>>,
6472
}
6573

6674
impl ModuleBatchesGraph {
@@ -79,6 +87,28 @@ impl ModuleBatchesGraph {
7987
Ok(*entry)
8088
}
8189

90+
pub fn get_ordered_entries<'l>(
91+
&'l self,
92+
chunk_group_info: &'l ChunkGroupInfo,
93+
idx: usize,
94+
) -> impl Iterator<Item = ResolvedVc<Box<dyn Module>>> + 'l {
95+
if let Some(ordered_entries) = self
96+
.ordered_entries
97+
.get(idx)
98+
.as_ref()
99+
.and_then(|o| o.as_ref())
100+
{
101+
if let Some(chunk_group) = chunk_group_info.chunk_groups.get_index(idx) {
102+
debug_assert_eq!(ordered_entries.len(), chunk_group.entries_count());
103+
}
104+
Either::Left(Either::Left(ordered_entries.iter().copied()))
105+
} else if let Some(chunk_group) = chunk_group_info.chunk_groups.get_index(idx) {
106+
Either::Right(chunk_group.entries())
107+
} else {
108+
Either::Left(Either::Right(std::iter::empty()))
109+
}
110+
}
111+
82112
pub fn get_batch_group(
83113
&self,
84114
module_or_batch: &ModuleOrBatch,
@@ -323,6 +353,8 @@ pub async fn compute_module_batches(
323353
let mut pre_batches = PreBatches::new();
324354
let mut queue: VecDeque<(ResolvedVc<Box<dyn Module>>, PreBatchIndex)> = VecDeque::new();
325355

356+
let mut chunk_group_indicies_with_merged_children = FxHashSet::default();
357+
326358
// Start with the entries
327359
for chunk_group in &chunk_group_info.chunk_groups {
328360
for entry in chunk_group.entries() {
@@ -340,6 +372,9 @@ pub async fn compute_module_batches(
340372
pre_batches.single_module_entries.insert(entry);
341373
}
342374
}
375+
if let Some(parent) = chunk_group.get_merged_parent() {
376+
chunk_group_indicies_with_merged_children.insert(parent);
377+
}
343378
}
344379

345380
let mut initial_pre_batch_items = 0;
@@ -360,6 +395,83 @@ pub async fn compute_module_batches(
360395
span.record("initial_pre_batch_items", initial_pre_batch_items);
361396
span.record("initial_pre_batches", pre_batches.batches.len());
362397

398+
// Figure out the order of all merged groups
399+
let mut ordered_entries: Vec<Option<EntriesList>> =
400+
vec![None; chunk_group_info.chunk_groups.len()];
401+
for (i, chunk_group) in chunk_group_info.chunk_groups.iter().enumerate() {
402+
if !chunk_group_indicies_with_merged_children.contains(&i) {
403+
continue;
404+
}
405+
let mut merged_modules: FxHashMap<ChunkingType, FxIndexSet<_>> = FxHashMap::default();
406+
let mut stack = ordered_entries[i]
407+
.as_ref()
408+
.map_or_else(
409+
|| Either::Left(chunk_group.entries()),
410+
|v| Either::Right(v.iter().copied()),
411+
)
412+
.filter_map(|module| {
413+
if let Some(chunkable_module) = ResolvedVc::try_downcast(module) {
414+
let idx = *pre_batches.entries.get(&chunkable_module).unwrap();
415+
Some((idx, 0))
416+
} else {
417+
None
418+
}
419+
})
420+
.collect::<Vec<_>>();
421+
stack.reverse();
422+
let mut visited = FxHashSet::default();
423+
while let Some((idx, mut pos)) = stack.pop() {
424+
let batch = &pre_batches.batches[idx];
425+
while let Some(item) = batch.items.get_index(pos) {
426+
match item {
427+
PreBatchItem::ParallelModule(_) => {}
428+
PreBatchItem::ParallelReference(other_idx) => {
429+
if visited.insert(*other_idx) {
430+
stack.push((idx, pos + 1));
431+
stack.push((*other_idx, 0));
432+
break;
433+
}
434+
}
435+
PreBatchItem::NonParallelEdge(chunking_type, module) => {
436+
if chunking_type.is_merged() {
437+
merged_modules
438+
.entry(chunking_type.clone())
439+
.or_default()
440+
.insert(*module);
441+
}
442+
}
443+
}
444+
pos += 1;
445+
}
446+
}
447+
if !merged_modules.is_empty() {
448+
for (ty, merged_modules) in merged_modules {
449+
let chunk_group_key = match ty {
450+
ChunkingType::Isolated {
451+
merge_tag: Some(merge_tag),
452+
..
453+
} => ChunkGroupKey::IsolatedMerged {
454+
parent: i.into(),
455+
merge_tag: merge_tag.clone(),
456+
},
457+
ChunkingType::Shared {
458+
merge_tag: Some(merge_tag),
459+
..
460+
} => ChunkGroupKey::SharedMerged {
461+
parent: i.into(),
462+
merge_tag: merge_tag.clone(),
463+
},
464+
_ => unreachable!(),
465+
};
466+
let idx = chunk_group_info
467+
.chunk_group_keys
468+
.get_index_of(&chunk_group_key)
469+
.unwrap();
470+
ordered_entries[idx] = Some(merged_modules);
471+
}
472+
}
473+
}
474+
363475
// Create a map of parallel module to the batches they are contained in.
364476
let mut parallel_module_to_pre_batch: FxIndexMap<_, Vec<PreBatchIndex>> =
365477
FxIndexMap::default();
@@ -748,6 +860,7 @@ pub async fn compute_module_batches(
748860
graph: TracedDiGraph(graph),
749861
entries,
750862
batch_groups,
863+
ordered_entries,
751864
}
752865
.cell())
753866
}

0 commit comments

Comments
 (0)