Lazy DevHtmlAsset chunk generation (vercel/turborepo#4679)

### Description

This fixes a perf regression introduced in the chunking refactor, where
we would be eagerly generating all assets from the web entry and
fallback sources.

### Testing Instructions

Benchmarks

[Next.js
PR](https://github.com/vercel/next.js/compare/alexkirsz/web-944-perf-regression?expand=1)

link WEB-944

Author: alexkirsz

crates/turbopack-cli/src/dev/web_entry_source.rs

@@ -254,20 +254,20 @@ pub async fn create_web_entry_source(
         .try_join()
         .await?;
 
-    let chunk_groups: Vec<_> = entries
+    let entries: Vec<_> = entries
         .into_iter()
         .flatten()
         .map(|module| async move {
             if let Some(ecmascript) = EcmascriptModuleAssetVc::resolve_from(module).await? {
-                let chunk_group = chunking_context.evaluated_chunk_group(
-                    ecmascript.as_root_chunk(chunking_context),
-                    runtime_entries.with_entry(ecmascript.into()),
-                );
-                Ok(chunk_group)
+                Ok((
+                    ecmascript.into(),
+                    chunking_context,
+                    Some(runtime_entries.with_entry(ecmascript.into())),
+                ))
             } else if let Some(chunkable) = ChunkableAssetVc::resolve_from(module).await? {
                 // TODO this is missing runtime code, so it's probably broken and we should also
                 // add an ecmascript chunk with the runtime code
-                Ok(chunking_context.chunk_group(chunkable.as_root_chunk(chunking_context)))
+                Ok((chunkable.into(), chunking_context, None))
             } else {
                 // TODO convert into a serve-able asset
                 Err(anyhow!(
@@ -279,7 +279,7 @@ pub async fn create_web_entry_source(
         .try_join()
         .await?;
 
-    let entry_asset = DevHtmlAssetVc::new(server_root.join("index.html"), chunk_groups).into();
+    let entry_asset = DevHtmlAssetVc::new(server_root.join("index.html"), entries).into();
 
     let graph = if eager_compile {
         AssetGraphContentSourceVc::new_eager(server_root, entry_asset)

crates/turbopack-dev-server/src/html.rs

@@ -1,10 +1,13 @@
 use anyhow::{anyhow, Result};
 use mime_guess::mime::TEXT_HTML_UTF_8;
-use turbo_tasks::primitives::StringVc;
+use turbo_tasks::{primitives::StringVc, TryJoinIterExt};
 use turbo_tasks_fs::{File, FileSystemPathVc};
 use turbo_tasks_hash::{encode_hex, Xxh3Hash64Hasher};
 use turbopack_core::{
     asset::{Asset, AssetContentVc, AssetVc, AssetsVc},
+    chunk::{
+        ChunkableAsset, ChunkableAssetVc, ChunkingContext, ChunkingContextVc, EvaluatableAssetsVc,
+    },
     ident::AssetIdentVc,
     reference::{AssetReferencesVc, SingleAssetReferenceVc},
     version::{Version, VersionVc, VersionedContent, VersionedContentVc},
@@ -17,7 +20,13 @@ use turbopack_core::{
 #[derive(Clone)]
 pub struct DevHtmlAsset {
     path: FileSystemPathVc,
-    chunk_groups: Vec<AssetsVc>,
+    // TODO(WEB-945) This should become a `Vec<DevHtmlEntry>` once we have a
+    // `turbo_tasks::input` attribute macro/`Input` derive macro.
+    entries: Vec<(
+        ChunkableAssetVc,
+        ChunkingContextVc,
+        Option<EvaluatableAssetsVc>,
+    )>,
     body: Option<String>,
 }
 
@@ -39,16 +48,12 @@ impl Asset for DevHtmlAsset {
     }
 
     #[turbo_tasks::function]
-    async fn references(&self) -> Result<AssetReferencesVc> {
+    async fn references(self_vc: DevHtmlAssetVc) -> Result<AssetReferencesVc> {
         let mut references = Vec::new();
-        for chunk_group in &self.chunk_groups {
-            let chunks = chunk_group.await?;
-            for chunk in chunks.iter() {
-                references.push(
-                    SingleAssetReferenceVc::new(*chunk, dev_html_chunk_reference_description())
-                        .into(),
-                );
-            }
+        for chunk in &*self_vc.chunks().await? {
+            references.push(
+                SingleAssetReferenceVc::new(*chunk, dev_html_chunk_reference_description()).into(),
+            );
         }
         Ok(AssetReferencesVc::cell(references))
     }
@@ -61,10 +66,17 @@ impl Asset for DevHtmlAsset {
 
 impl DevHtmlAssetVc {
     /// Create a new dev HTML asset.
-    pub fn new(path: FileSystemPathVc, chunk_groups: Vec<AssetsVc>) -> Self {
+    pub fn new(
+        path: FileSystemPathVc,
+        entries: Vec<(
+            ChunkableAssetVc,
+            ChunkingContextVc,
+            Option<EvaluatableAssetsVc>,
+        )>,
+    ) -> Self {
         DevHtmlAsset {
             path,
-            chunk_groups,
+            entries,
             body: None,
         }
         .cell()
@@ -73,12 +85,16 @@ impl DevHtmlAssetVc {
     /// Create a new dev HTML asset.
     pub fn new_with_body(
         path: FileSystemPathVc,
-        chunk_groups: Vec<AssetsVc>,
+        entries: Vec<(
+            ChunkableAssetVc,
+            ChunkingContextVc,
+            Option<EvaluatableAssetsVc>,
+        )>,
         body: String,
     ) -> Self {
         DevHtmlAsset {
             path,
-            chunk_groups,
+            entries,
             body: Some(body),
         }
         .cell()
@@ -110,17 +126,44 @@ impl DevHtmlAssetVc {
         let context_path = this.path.parent().await?;
 
         let mut chunk_paths = vec![];
-        for chunk_group in &this.chunk_groups {
-            for chunk in chunk_group.await?.iter() {
-                let chunk_path = &*chunk.ident().path().await?;
-                if let Some(relative_path) = context_path.get_path_to(chunk_path) {
-                    chunk_paths.push(format!("/{relative_path}"));
-                }
+        for chunk in &*self.chunks().await? {
+            let chunk_path = &*chunk.ident().path().await?;
+            if let Some(relative_path) = context_path.get_path_to(chunk_path) {
+                chunk_paths.push(format!("/{relative_path}"));
             }
         }
 
         Ok(DevHtmlAssetContentVc::new(chunk_paths, this.body.clone()))
     }
+
+    #[turbo_tasks::function]
+    async fn chunks(self) -> Result<AssetsVc> {
+        let this = self.await?;
+
+        let all_assets = this
+            .entries
+            .iter()
+            .map(|entry| async move {
+                let (chunkable_asset, chunking_context, runtime_entries) = entry;
+
+                let chunk = chunkable_asset.as_root_chunk(*chunking_context);
+                let assets = if let Some(runtime_entries) = runtime_entries {
+                    chunking_context.evaluated_chunk_group(chunk, *runtime_entries)
+                } else {
+                    chunking_context.chunk_group(chunk)
+                };
+
+                Ok(assets.await?)
+            })
+            .try_join()
+            .await?
+            .iter()
+            .flatten()
+            .copied()
+            .collect();
+
+        Ok(AssetsVc::cell(all_assets))
+    }
 }
 
 #[turbo_tasks::value]