From b4338c3b4037cca12cc9502c0ea7fdfbb1faf6ca Mon Sep 17 00:00:00 2001 From: Benjamin Woodruff Date: Tue, 9 Jul 2024 12:06:15 -0700 Subject: [PATCH] Write out task statistics if NEXT_TURBOPACK_TASK_STATISTICS is set (#67164) Writes out the statistics gathered via vercel/turbo#8286 to a file. Stats can be formatted and sorted with jq: ``` jq 'to_entries | sort_by(.value.cache_miss) | reverse | from_entries' output.json ``` Output looks something like this (once formatted and sorted): https://gist.github.com/bgw/4e2df35b9e410bf71fe51ecaffc3c7bf Without #67165, this requires that SIGINT be sent directly to the `next-server` process to allow a clean exit: ``` pkill -INT next-server ``` But with #67165, a simple ctrl+c works. --- .../crates/napi/src/next_api/project.rs | 54 ++++++++++++++----- packages/next/src/build/swc/index.ts | 6 +++ .../src/server/dev/hot-reloader-turbopack.ts | 1 + packages/next/src/server/lib/router-server.ts | 2 + .../lib/router-utils/setup-dev-bundler.ts | 1 + packages/next/src/server/lib/start-server.ts | 10 +++- packages/next/src/server/next.ts | 13 ++++- 7 files changed, 70 insertions(+), 17 deletions(-) diff --git a/packages/next-swc/crates/napi/src/next_api/project.rs b/packages/next-swc/crates/napi/src/next_api/project.rs index 65a4d59d9602b..57b6536639339 100644 --- a/packages/next-swc/crates/napi/src/next_api/project.rs +++ b/packages/next-swc/crates/napi/src/next_api/project.rs @@ -1,4 +1,4 @@ -use std::{path::PathBuf, sync::Arc, thread, time::Duration}; +use std::{io::Write, path::PathBuf, sync::Arc, thread, time::Duration}; use anyhow::{anyhow, bail, Context, Result}; use napi::{ @@ -39,9 +39,9 @@ use turbopack_binding::{ }, ecmascript_hmr_protocol::{ClientUpdateInstruction, ResourceIdentifier}, trace_utils::{ - exit::ExitGuard, + exit::{ExitHandler, ExitReceiver}, raw_trace::RawTraceLayer, - trace_writer::{TraceWriter, TraceWriterGuard}, + trace_writer::TraceWriter, }, }, }; @@ -252,8 +252,7 @@ impl From for DefineEnv { pub struct ProjectInstance { turbo_tasks: Arc>, container: Vc, - #[allow(dead_code)] - guard: Option>, + exit_receiver: tokio::sync::Mutex>, } #[napi(ts_return_type = "{ __napiType: \"Project\" }")] @@ -264,8 +263,9 @@ pub async fn project_new( register(); let trace = std::env::var("NEXT_TURBOPACK_TRACING").ok(); + let (exit, exit_receiver) = ExitHandler::new_receiver(); - let guard = if let Some(mut trace) = trace { + if let Some(mut trace) = trace { // Trace presets match trace.as_str() { "overview" | "1" => { @@ -297,10 +297,12 @@ pub async fn project_new( .unwrap(); let trace_file = internal_dir.join("trace.log"); let trace_writer = std::fs::File::create(trace_file.clone()).unwrap(); - let (trace_writer, guard) = TraceWriter::new(trace_writer); + let (trace_writer, trace_writer_guard) = TraceWriter::new(trace_writer); let subscriber = subscriber.with(RawTraceLayer::new(trace_writer)); - let guard = ExitGuard::new(guard).unwrap(); + exit.on_exit(async move { + tokio::task::spawn_blocking(move || drop(trace_writer_guard)); + }); let trace_server = std::env::var("NEXT_TURBOPACK_TRACE_SERVER").ok(); if trace_server.is_some() { @@ -313,11 +315,7 @@ pub async fn project_new( } subscriber.init(); - - Some(guard) - } else { - None - }; + } let turbo_tasks = TurboTasks::new(MemoryBackend::new( turbo_engine_options @@ -325,6 +323,22 @@ pub async fn project_new( .map(|m| m as usize) .unwrap_or(usize::MAX), )); + let stats_path = std::env::var_os("NEXT_TURBOPACK_TASK_STATISTICS"); + if let Some(stats_path) = stats_path { + let task_stats = turbo_tasks.backend().task_statistics().enable().clone(); + exit.on_exit(async move { + tokio::task::spawn_blocking(move || { + let mut file = std::fs::File::create(&stats_path) + .with_context(|| format!("failed to create or open {stats_path:?}"))?; + serde_json::to_writer(&file, &task_stats) + .context("failed to serialize or write task statistics")?; + file.flush().context("failed to flush file") + }) + .await + .unwrap() + .unwrap(); + }); + } let options = options.into(); let container = turbo_tasks .run_once(async move { @@ -344,7 +358,7 @@ pub async fn project_new( ProjectInstance { turbo_tasks, container, - guard, + exit_receiver: tokio::sync::Mutex::new(Some(exit_receiver)), }, 100, )) @@ -1100,3 +1114,15 @@ pub async fn project_get_source_for_asset( Ok(source) } + +/// Runs exit handlers for the project registered using the [`ExitHandler`] API. +#[napi] +pub async fn project_on_exit( + #[napi(ts_arg_type = "{ __napiType: \"Project\" }")] project: External, +) { + let exit_receiver = project.exit_receiver.lock().await.take(); + exit_receiver + .expect("`project.onExitSync` must only be called once") + .run_exit_handler() + .await; +} diff --git a/packages/next/src/build/swc/index.ts b/packages/next/src/build/swc/index.ts index cdca6a501760c..6882a1a23c0c5 100644 --- a/packages/next/src/build/swc/index.ts +++ b/packages/next/src/build/swc/index.ts @@ -663,6 +663,8 @@ export interface Project { updateInfoSubscribe( aggregationMs: number ): AsyncIterableIterator> + + onExit(): Promise } export type Route = @@ -1074,6 +1076,10 @@ function bindingToApi( ) return subscription } + + onExit(): Promise { + return binding.projectOnExit(this._nativeProject) + } } class EndpointImpl implements Endpoint { diff --git a/packages/next/src/server/dev/hot-reloader-turbopack.ts b/packages/next/src/server/dev/hot-reloader-turbopack.ts index 00807c847a890..70267f8550eab 100644 --- a/packages/next/src/server/dev/hot-reloader-turbopack.ts +++ b/packages/next/src/server/dev/hot-reloader-turbopack.ts @@ -153,6 +153,7 @@ export async function createHotReloaderTurbopack( memoryLimit: opts.nextConfig.experimental.turbo?.memoryLimit, } ) + opts.onCleanup(() => project.onExit()) const entrypointsSubscription = project.entrypointsSubscribe() const currentEntrypoints: Entrypoints = { diff --git a/packages/next/src/server/lib/router-server.ts b/packages/next/src/server/lib/router-server.ts index 0f0c8187ef798..b45f341d036bc 100644 --- a/packages/next/src/server/lib/router-server.ts +++ b/packages/next/src/server/lib/router-server.ts @@ -64,6 +64,7 @@ export async function initialize(opts: { dir: string port: number dev: boolean + onCleanup: (listener: () => Promise) => void server?: import('http').Server minimalMode?: boolean hostname?: string @@ -131,6 +132,7 @@ export async function initialize(opts: { isCustomServer: opts.customServer, turbo: !!process.env.TURBOPACK, port: opts.port, + onCleanup: opts.onCleanup, }) ) diff --git a/packages/next/src/server/lib/router-utils/setup-dev-bundler.ts b/packages/next/src/server/lib/router-utils/setup-dev-bundler.ts index 175ea8b60d5ce..d5dd11d90993e 100644 --- a/packages/next/src/server/lib/router-utils/setup-dev-bundler.ts +++ b/packages/next/src/server/lib/router-utils/setup-dev-bundler.ts @@ -96,6 +96,7 @@ export type SetupOpts = { > nextConfig: NextConfigComplete port: number + onCleanup: (listener: () => Promise) => void } export type ServerFields = { diff --git a/packages/next/src/server/lib/start-server.ts b/packages/next/src/server/lib/start-server.ts index 13f20677acce0..5c2d541e17aab 100644 --- a/packages/next/src/server/lib/start-server.ts +++ b/packages/next/src/server/lib/start-server.ts @@ -46,6 +46,7 @@ export async function getRequestHandlers({ dir, port, isDev, + onCleanup, server, hostname, minimalMode, @@ -56,6 +57,7 @@ export async function getRequestHandlers({ dir: string port: number isDev: boolean + onCleanup: (listener: () => Promise) => void server?: import('http').Server hostname?: string minimalMode?: boolean @@ -67,6 +69,7 @@ export async function getRequestHandlers({ dir, port, hostname, + onCleanup, dev: isDev, minimalMode, server, @@ -262,9 +265,13 @@ export async function startServer( Log.event(`Starting...`) try { + const cleanupListeners = [() => new Promise((res) => server.close(res))] const cleanup = () => { debug('start-server process cleanup') - server.close(() => process.exit(0)) + ;(async () => { + await Promise.all(cleanupListeners.map((f) => f())) + process.exit(0) + })() } const exception = (err: Error) => { if (isPostpone(err)) { @@ -294,6 +301,7 @@ export async function startServer( dir, port, isDev, + onCleanup: (listener) => cleanupListeners.push(listener), server, hostname, minimalMode, diff --git a/packages/next/src/server/next.ts b/packages/next/src/server/next.ts index 6b2b8b7e4cb25..bfd02a85c1546 100644 --- a/packages/next/src/server/next.ts +++ b/packages/next/src/server/next.ts @@ -63,6 +63,7 @@ export class NextServer { private reqHandlerPromise?: Promise private preparedAssetPrefix?: string + protected cleanupListeners: (() => Promise)[] = [] protected standaloneMode?: boolean public options: NextServerOptions @@ -156,8 +157,15 @@ export class NextServer { } async close() { - const server = await this.getServer() - return (server as any).close() + await Promise.all( + [ + async () => { + const server = await this.getServer() + await (server as any).close() + }, + ...this.cleanupListeners, + ].map((f) => f()) + ) } private async createServer( @@ -283,6 +291,7 @@ class NextCustomServer extends NextServer { dir: this.options.dir!, port: this.options.port || 3000, isDev: !!this.options.dev, + onCleanup: (listener) => this.cleanupListeners.push(listener), hostname: this.options.hostname || 'localhost', minimalMode: this.options.minimalMode, isNodeDebugging: !!isNodeDebugging,