Generic types over Vcs (vercel/turborepo#5738)

### Description

This PR enables types generic over `Vc`s to work as value types.

In other words, where we would previously need to declare all
generic/container types over a given value type at the definition site,
like the following:

```rust
#[turbo_tasks::value]
struct Thing;

#[turbo_tasks::value(transparent)]
struct Things(Vec<Vc<Thing>>);

#[turbo_tasks::value(transparent)]
struct ThingOption(Option<Vc<Thing>>);

#[turbo_tasks::value(transparent)]
struct ThingSet(IndexSet<Vc<Thing>>);

#[turbo_tasks::value(transparent)]
struct ThingMap(IndexMap<Vc<ThingKey>, Vc<Thing>>);

let vec: Vc<Things> = Vc::cell(vec![Thing.cell()]);
let option: Vc<ThingOption> = Vc::cell(Some(Thing.cell()));

// etc.
```

We can now refer to `Vc<Vec<Vc<Thing>>>` directly, without needing to
declare an intermediate transparent value type:

```rust
#[turbo_tasks::value]
struct Thing;

let vec: Vc<Vec<_>> = Vc::cell(vec![Thing.cell()]);
let option: Vc<Option<_>> = Vc::cell(Some(Thing.cell()));
// etc.
```

The following generic types and methods are currently supported:
* `Option<Vc<T>>`
  * `::is_some(self: Vc<Self>) -> Vc<bool>`
  * `::is_none(self: Vc<Self>) -> Vc<bool>`
* `Vec<Vc<T>>`
  * `::is_empty(self: Vc<Self>) -> Vc<bool>`
  * `::len(self: Vc<Self>) -> Vc<usize>`
* `IndexSet<Vc<T>>`
  * `::is_empty(self: Vc<Self>) -> Vc<bool>`
  * `::len(self: Vc<Self>) -> Vc<usize>`
* `IndexMap<Vc<K>, Vc<V>>`
  * `::is_empty(self: Vc<Self>) -> Vc<bool>`
  * `::len(self: Vc<Self>) -> Vc<usize>`

All these value types also implement `ValueDefault`, which means that
you can instantiate them using `Default::default()` or `Vc::default()`:

```rust
let vec: Vc<Vec<Vc<u32>>> = Default::default()
```

They also work recursively, but I would recommend creating intermediate
types instead of this in most (all?) cases:

```rust
let vec: Vc<Vec<Vc<Vec<Vc<u32>>>>> = Default::default()
```

### Notes and implementation details

1. This only works when the generic type is wrapped in a `Vc`. In the
example above, this would not cover `Vc<Vec<Thing>>` directly. The
implementation relies on the fact that all `Type<Vc<T>>` are represented
the exact same way in memory, and as such can all be cast to some common
`Type<Vc<()>>` for storage.
2. Like primitives, these generic types can currently only be declared
in `turbo_tasks` itself. Technically, if a crate owns a type `A`, it
could also register `A` as a generic type, so we might want to allow
this in the future (e.g. `AliasMap` in `turbopack`?)
3. There's some potential performance/correctness issues around
`cell`ing, I'll defer to @sokra to figure out if this approach is worth
it.

### Testing Instructions

I added some tests to `all_in_one.rs`.

---------

Co-authored-by: Justin Ridgewell <justin@ridgewell.name>
Co-authored-by: Tobias Koppers <tobias.koppers@googlemail.com>

crates/turbo-tasks-build/src/lib.rs

@@ -18,7 +18,7 @@ use turbo_tasks_macros_shared::{
     get_impl_function_ident, get_native_function_ident, get_path_ident,
     get_register_trait_methods_ident, get_register_value_type_ident,
     get_trait_default_impl_function_ident, get_trait_impl_function_ident, get_trait_type_ident,
-    get_type_ident, PrimitiveInput, ValueTraitArguments,
+    get_type_ident, GenericTypeInput, PrimitiveInput, ValueTraitArguments,
 };
 
 pub fn generate_register() {
@@ -212,6 +212,7 @@ impl<'a> RegisterContext<'a> {
     fn process_enum(&mut self, enum_item: ItemEnum) -> Result<()> {
         if has_attribute(&enum_item.attrs, "value") {
             self.add_value(&enum_item.ident);
+            self.add_value_debug_impl(&enum_item.ident);
         }
         Ok(())
     }
@@ -286,6 +287,7 @@ impl<'a> RegisterContext<'a> {
     fn process_struct(&mut self, struct_item: ItemStruct) -> Result<()> {
         if has_attribute(&struct_item.attrs, "value") {
             self.add_value(&struct_item.ident);
+            self.add_value_debug_impl(&struct_item.ident);
         }
         Ok(())
     }
@@ -347,6 +349,23 @@ impl<'a> RegisterContext<'a> {
             };
 
             self.add_value(&ident);
+            self.add_value_debug_impl(&ident);
+        } else if macro_item
+            .mac
+            .path
+            .is_ident("__turbo_tasks_internal_generic_type")
+        {
+            let input = macro_item.mac.tokens;
+            let input = syn::parse2::<GenericTypeInput>(input).unwrap();
+
+            let ty = input.ty;
+            let Some(ident) = get_type_ident(&ty) else {
+                return Ok(());
+            };
+
+            // Generic types must implement `ValueDebug` manually, as there's currently no
+            // easy way to automate the process.
+            self.add_value(&ident);
         }
 
         Ok(())
@@ -376,7 +395,9 @@ impl<'a> RegisterContext<'a> {
             "{} is declared more than once",
             ident
         );
+    }
 
+    fn add_value_debug_impl(&mut self, ident: &Ident) {
         // register default debug impl generated by proc macro
         self.register_debug_impl(ident).unwrap();
         self.add_value_trait(

crates/turbo-tasks-macros-shared/src/generic_type_input.rs

@@ -0,0 +1,21 @@
+use syn::{
+    parse::{Parse, ParseStream},
+    Generics, Result, Token, Type,
+};
+
+/// The input of the `generic_type` macro.
+#[derive(Debug)]
+pub struct GenericTypeInput {
+    pub generics: Generics,
+    pub ty: Type,
+}
+
+impl Parse for GenericTypeInput {
+    fn parse(input: ParseStream) -> Result<Self> {
+        let generics: Generics = input.parse()?;
+        let _comma: Token![,] = input.parse()?;
+        let ty: Type = input.parse()?;
+
+        Ok(GenericTypeInput { generics, ty })
+    }
+}

crates/turbo-tasks-macros-shared/src/lib.rs

@@ -2,11 +2,13 @@
 #![feature(box_patterns)]
 
 mod expand;
+mod generic_type_input;
 mod ident;
 mod primitive_input;
 mod value_trait_arguments;
 
 pub use expand::*;
+pub use generic_type_input::GenericTypeInput;
 pub use ident::*;
 pub use primitive_input::PrimitiveInput;
 pub use value_trait_arguments::ValueTraitArguments;

crates/turbo-tasks-macros/Cargo.toml

@@ -15,5 +15,5 @@ convert_case = "0.6.0"
 proc-macro-error = "1.0.4"
 proc-macro2 = { workspace = true }
 quote = { workspace = true }
-syn = { workspace = true, features = ["full", "extra-traits"] }
+syn = { workspace = true, features = ["full", "extra-traits", "visit-mut"] }
 turbo-tasks-macros-shared = { workspace = true }

crates/turbo-tasks-macros/src/generic_type_macro.rs

@@ -0,0 +1,148 @@
+use proc_macro::TokenStream;
+use quote::quote;
+use syn::{parse_macro_input, spanned::Spanned, visit_mut::VisitMut, GenericParam, Lifetime, Type};
+use turbo_tasks_macros_shared::{get_type_ident, GenericTypeInput};
+
+use crate::value_macro::value_type_and_register;
+
+pub fn generic_type(input: TokenStream) -> TokenStream {
+    let input = parse_macro_input!(input as GenericTypeInput);
+
+    for param in &input.generics.params {
+        match param {
+            syn::GenericParam::Type(ty) => {
+                if ty.ident == "Vc" {
+                    ty.span()
+                        .unwrap()
+                        .error("Vc is a reserved name in generic_type")
+                        .emit();
+                }
+            }
+            syn::GenericParam::Lifetime(lt) => {
+                lt.span()
+                    .unwrap()
+                    .error("lifetime parameters are not supported in generic_type")
+                    .emit();
+            }
+            syn::GenericParam::Const(c) => {
+                c.span()
+                    .unwrap()
+                    .error("const parameters are not supported in generic_type")
+                    .emit();
+            }
+        }
+    }
+
+    let (impl_generics, _, where_clause) = input.generics.split_for_impl();
+
+    let repr = replace_generics_with_unit(input.generics.params.iter(), &input.ty);
+
+    let ty = input.ty;
+    let Some(ident) = get_type_ident(&ty) else {
+        return quote! {
+            // An error occurred while parsing the ident.
+        }
+        .into();
+    };
+
+    let mut generics_with_static = input.generics.clone();
+    for param in &mut generics_with_static.params {
+        if let GenericParam::Type(param) = param {
+            param.bounds.push(syn::TypeParamBound::Lifetime(Lifetime {
+                ident: syn::Ident::new("static", param.ident.span()),
+                apostrophe: param.ident.span(),
+            }))
+        }
+    }
+
+    let value_type_and_register = value_type_and_register(
+        &ident,
+        quote! { #ty },
+        Some(&generics_with_static),
+        quote! {
+            turbo_tasks::VcTransparentRead<#ty, #ty, #repr>
+        },
+        quote! {
+            turbo_tasks::VcCellSharedMode<#ty>
+        },
+        quote! {
+            turbo_tasks::ValueType::new_with_any_serialization::<#repr>()
+        },
+    );
+
+    quote! {
+        #value_type_and_register
+
+        impl #impl_generics Vc<#ty> #where_clause {
+            /// Converts this `Vc` to a generic representation.
+            fn to_repr(vc: Self) -> Vc<#repr> {
+                unsafe {
+                    turbo_tasks::Vc::from_raw(Vc::into_raw(vc))
+                }
+            }
+
+            /// Converts a generic representation of this `Vc` to the proper `Vc` type.
+            ///
+            /// # Safety
+            ///
+            /// The caller must ensure that the `repr` is a valid representation of this `Vc`.
+            unsafe fn from_repr(vc: Vc<#repr>) -> Self {
+                unsafe {
+                    turbo_tasks::Vc::from_raw(Vc::into_raw(vc))
+                }
+            }
+        }
+    }
+    .into()
+}
+
+struct ReplaceGenericsVisitor<'a> {
+    generics: &'a std::collections::HashSet<String>,
+}
+
+impl<'a> VisitMut for ReplaceGenericsVisitor<'a> {
+    fn visit_type_mut(&mut self, node: &mut Type) {
+        if let Type::Path(type_path) = node {
+            if type_path.qself.is_none()
+                && type_path.path.segments.len() == 1
+                && type_path.path.segments[0].arguments.is_none()
+                && self
+                    .generics
+                    .contains(&type_path.path.segments[0].ident.to_string())
+            {
+                // Replace the whole path with ()
+                *node = syn::parse_quote! { () };
+                return;
+            }
+        }
+
+        syn::visit_mut::visit_type_mut(self, node);
+    }
+}
+
+/// Replaces all instances of `params` generic types in `ty` with the unit type
+/// `()`.
+fn replace_generics_with_unit<'a, P>(params: P, ty: &Type) -> Type
+where
+    P: IntoIterator<Item = &'a GenericParam>,
+{
+    let generics_set: std::collections::HashSet<_> = params
+        .into_iter()
+        .filter_map(|param| {
+            if let GenericParam::Type(type_param) = param {
+                Some(type_param.ident.to_string())
+            } else {
+                None
+            }
+        })
+        .collect();
+
+    let mut new_ty = ty.clone();
+    let mut visitor = ReplaceGenericsVisitor {
+        generics: &generics_set,
+    };
+
+    syn::visit_mut::visit_type_mut(&mut visitor, &mut new_ty);
+
+    new_ty
+}

crates/turbo-tasks-macros/src/lib.rs

@@ -5,6 +5,7 @@
 mod derive;
 mod func;
 mod function_macro;
+mod generic_type_macro;
 mod primitive_macro;
 mod value_impl_macro;
 mod value_macro;
@@ -147,3 +148,28 @@ pub fn value_impl(args: TokenStream, input: TokenStream) -> TokenStream {
 pub fn primitive(input: TokenStream) -> TokenStream {
     primitive_macro::primitive(input)
 }
+
+/// Registers a value type that is generic over the `Vc` it contains.
+///
+/// # Example
+///
+/// ```
+/// use crate::generic_type as __turbo_tasks_internal_generic_type;
+///
+/// __turbo_tasks_internal_generic_type!(<A, B>, GenericType<Vc<A>, Vc<B>>);
+///
+/// // Now you can do the following, for any `A` and `B` value types:
+///
+/// let vc: Vc<GenericType<Vc<u32>, Vc<String>>> = Vc::cell(
+///     GenericType::new(
+///         Vc::cell(42),
+///         Vc::cell("hello".to_string())
+///     )
+/// );
+/// ```
+#[allow_internal_unstable(min_specialization, into_future, trivial_bounds)]
+#[proc_macro_error]
+#[proc_macro]
+pub fn generic_type(input: TokenStream) -> TokenStream {
+    generic_type_macro::generic_type(input)
+}

crates/turbo-tasks-macros/src/primitive_macro.rs

@@ -36,8 +36,9 @@ pub fn primitive(input: TokenStream) -> TokenStream {
     let value_type_and_register = value_type_and_register(
         &ident,
         quote! { #ty },
+        None,
         quote! {
-            turbo_tasks::VcTransparentRead<#ty, #ty>
+            turbo_tasks::VcTransparentRead<#ty, #ty, #ty>
         },
         quote! {
             turbo_tasks::VcCellSharedMode<#ty>

crates/turbo-tasks-macros/src/value_impl_macro.rs

@@ -6,7 +6,7 @@ use syn::{
     parse_macro_input, parse_quote,
     punctuated::Punctuated,
     spanned::Spanned,
-    Attribute, Error, ExprPath, ImplItem, ImplItemMethod, ItemImpl, Lit, LitStr, Meta,
+    Attribute, Error, ExprPath, Generics, ImplItem, ImplItemMethod, ItemImpl, Lit, LitStr, Meta,
     MetaNameValue, Path, Result, Token, Type,
 };
 use turbo_tasks_macros_shared::{
@@ -158,9 +158,9 @@ pub fn value_impl(args: TokenStream, input: TokenStream) -> TokenStream {
                     }
 
                     #[doc(hidden)]
-                    pub(crate) static #native_function_ident: #native_function_ty = #ty::#native_function_ident;
+                    pub(crate) static #native_function_ident: #native_function_ty = <#ty>::#native_function_ident;
                     #[doc(hidden)]
-                    pub(crate) static #native_function_id_ident: #native_function_id_ty = #ty::#native_function_id_ident;
+                    pub(crate) static #native_function_id_ident: #native_function_id_ty = <#ty>::#native_function_id_ident;
                 })
             }
         }
@@ -176,12 +176,15 @@ pub fn value_impl(args: TokenStream, input: TokenStream) -> TokenStream {
 
     fn trait_value_impl(
         ty: &Type,
+        generics: &Generics,
         ty_ident: &Ident,
         trait_path: &Path,
         items: &[ImplItem],
     ) -> TokenStream2 {
         let trait_ident = get_path_ident(trait_path);
 
+        let (impl_generics, _, where_clause) = generics.split_for_impl();
+
         let register = get_register_trait_methods_ident(&trait_ident, ty_ident);
 
         let mut trait_registers = Vec::new();
@@ -263,7 +266,7 @@ pub fn value_impl(args: TokenStream, input: TokenStream) -> TokenStream {
 
                     #[doc(hidden)]
                     // #[turbo_tasks::async_trait]
-                    impl #inline_extension_trait_ident for #ty {
+                    impl #impl_generics #inline_extension_trait_ident for #ty #where_clause  {
                         #[allow(declare_interior_mutable_const)]
                         #[doc(hidden)]
                         const #native_function_ident: #native_function_ty = #native_function_def;
@@ -300,9 +303,9 @@ pub fn value_impl(args: TokenStream, input: TokenStream) -> TokenStream {
             // NOTE(alexkirsz) We can't have a general `turbo_tasks::Upcast<Box<dyn Trait>> for T where T: Trait` because
             // rustc complains: error[E0210]: type parameter `T` must be covered by another type when it appears before
             // the first local type (`dyn Trait`).
-            unsafe impl turbo_tasks::Upcast<Box<dyn #trait_path>> for #ty {}
+            unsafe impl #impl_generics turbo_tasks::Upcast<Box<dyn #trait_path>> for #ty #where_clause {}
 
-            impl #trait_path for #ty {
+            impl #impl_generics #trait_path for #ty #where_clause {
                 #(#trait_functions)*
             }
 
@@ -324,8 +327,13 @@ pub fn value_impl(args: TokenStream, input: TokenStream) -> TokenStream {
 
     match &item.trait_ {
         None => inherent_value_impl(&item.self_ty, &ty_ident, &item.items).into(),
-        Some((_, trait_path, _)) => {
-            trait_value_impl(&item.self_ty, &ty_ident, trait_path, &item.items).into()
-        }
+        Some((_, trait_path, _)) => trait_value_impl(
+            &item.self_ty,
+            &item.generics,
+            &ty_ident,
+            trait_path,
+            &item.items,
+        )
+        .into(),
     }
 }