Generic types over Vcs (vercel/turborepo#5738)

### Description

This PR enables types generic over `Vc`s to work as value types.

In other words, where we would previously need to declare all
generic/container types over a given value type at the definition site,
like the following:

```rust
#[turbo_tasks::value]
struct Thing;

#[turbo_tasks::value(transparent)]
struct Things(Vec<Vc<Thing>>);

#[turbo_tasks::value(transparent)]
struct ThingOption(Option<Vc<Thing>>);

#[turbo_tasks::value(transparent)]
struct ThingSet(IndexSet<Vc<Thing>>);

#[turbo_tasks::value(transparent)]
struct ThingMap(IndexMap<Vc<ThingKey>, Vc<Thing>>);

let vec: Vc<Things> = Vc::cell(vec![Thing.cell()]);
let option: Vc<ThingOption> = Vc::cell(Some(Thing.cell()));

// etc.
```

We can now refer to `Vc<Vec<Vc<Thing>>>` directly, without needing to
declare an intermediate transparent value type:

```rust
#[turbo_tasks::value]
struct Thing;

let vec: Vc<Vec<_>> = Vc::cell(vec![Thing.cell()]);
let option: Vc<Option<_>> = Vc::cell(Some(Thing.cell()));
// etc.
```

The following generic types and methods are currently supported:
* `Option<Vc<T>>`
  * `::is_some(self: Vc<Self>) -> Vc<bool>`
  * `::is_none(self: Vc<Self>) -> Vc<bool>`
* `Vec<Vc<T>>`
  * `::is_empty(self: Vc<Self>) -> Vc<bool>`
  * `::len(self: Vc<Self>) -> Vc<usize>`
* `IndexSet<Vc<T>>`
  * `::is_empty(self: Vc<Self>) -> Vc<bool>`
  * `::len(self: Vc<Self>) -> Vc<usize>`
* `IndexMap<Vc<K>, Vc<V>>`
  * `::is_empty(self: Vc<Self>) -> Vc<bool>`
  * `::len(self: Vc<Self>) -> Vc<usize>`

All these value types also implement `ValueDefault`, which means that
you can instantiate them using `Default::default()` or `Vc::default()`:

```rust
let vec: Vc<Vec<Vc<u32>>> = Default::default()
```

They also work recursively, but I would recommend creating intermediate
types instead of this in most (all?) cases:

```rust
let vec: Vc<Vec<Vc<Vec<Vc<u32>>>>> = Default::default()
```

### Notes and implementation details

1. This only works when the generic type is wrapped in a `Vc`. In the
example above, this would not cover `Vc<Vec<Thing>>` directly. The
implementation relies on the fact that all `Type<Vc<T>>` are represented
the exact same way in memory, and as such can all be cast to some common
`Type<Vc<()>>` for storage.
2. Like primitives, these generic types can currently only be declared
in `turbo_tasks` itself. Technically, if a crate owns a type `A`, it
could also register `A` as a generic type, so we might want to allow
this in the future (e.g. `AliasMap` in `turbopack`?)
3. There's some potential performance/correctness issues around
`cell`ing, I'll defer to @sokra to figure out if this approach is worth
it.

### Testing Instructions

I added some tests to `all_in_one.rs`.

---------

Co-authored-by: Justin Ridgewell <justin@ridgewell.name>
Co-authored-by: Tobias Koppers <tobias.koppers@googlemail.com>

crates/turbo-tasks-build/src/lib.rs

@@ -18,7 +18,7 @@ use turbo_tasks_macros_shared::{
  get_impl_function_ident, get_native_function_ident, get_path_ident,
  get_register_trait_methods_ident, get_register_value_type_ident,
  get_trait_default_impl_function_ident, get_trait_impl_function_ident, get_trait_type_ident,
- get_type_ident, PrimitiveInput, ValueTraitArguments,
+ get_type_ident, GenericTypeInput, PrimitiveInput, ValueTraitArguments,
 };
 
 pub fn generate_register() {
@@ -212,6 +212,7 @@ impl<'a> RegisterContext<'a> {
  fn process_enum(&mut self, enum_item: ItemEnum) -> Result<()> {
  if has_attribute(&enum_item.attrs, "value") {
  self.add_value(&enum_item.ident);
+ self.add_value_debug_impl(&enum_item.ident);
  }
  Ok(())
  }
@@ -286,6 +287,7 @@ impl<'a> RegisterContext<'a> {
  fn process_struct(&mut self, struct_item: ItemStruct) -> Result<()> {
  if has_attribute(&struct_item.attrs, "value") {
  self.add_value(&struct_item.ident);
+ self.add_value_debug_impl(&struct_item.ident);
  }
  Ok(())
  }
@@ -347,6 +349,23 @@ impl<'a> RegisterContext<'a> {
  };
 
  self.add_value(&ident);
+ self.add_value_debug_impl(&ident);
+ } else if macro_item
+ .mac
+ .path
+ .is_ident("__turbo_tasks_internal_generic_type")
+ {
+ let input = macro_item.mac.tokens;
+ let input = syn::parse2::<GenericTypeInput>(input).unwrap();
+
+ let ty = input.ty;
+ let Some(ident) = get_type_ident(&ty) else {
+ return Ok(());
+ };
+
+ // Generic types must implement `ValueDebug` manually, as there's currently no
+ // easy way to automate the process.
+ self.add_value(&ident);
  }
 
  Ok(())
@@ -376,7 +395,9 @@ impl<'a> RegisterContext<'a> {
  "{} is declared more than once",
  ident
  );
+ }
 
+ fn add_value_debug_impl(&mut self, ident: &Ident) {
  // register default debug impl generated by proc macro
  self.register_debug_impl(ident).unwrap();
  self.add_value_trait(

crates/turbo-tasks-macros-shared/src/generic_type_input.rs

@@ -0,0 +1,21 @@
+use syn::{
+ parse::{Parse, ParseStream},
+ Generics, Result, Token, Type,
+};
+
+/// The input of the `generic_type` macro.
+#[derive(Debug)]
+pub struct GenericTypeInput {
+ pub generics: Generics,
+ pub ty: Type,
+}
+
+impl Parse for GenericTypeInput {
+ fn parse(input: ParseStream) -> Result<Self> {
+ let generics: Generics = input.parse()?;
+ let _comma: Token![,] = input.parse()?;
+ let ty: Type = input.parse()?;
+
+ Ok(GenericTypeInput { generics, ty })
+ }
+}

crates/turbo-tasks-macros-shared/src/lib.rs

@@ -2,11 +2,13 @@
 #![feature(box_patterns)]
 
 mod expand;
+mod generic_type_input;
 mod ident;
 mod primitive_input;
 mod value_trait_arguments;
 
 pub use expand::*;
+pub use generic_type_input::GenericTypeInput;
 pub use ident::*;
 pub use primitive_input::PrimitiveInput;
 pub use value_trait_arguments::ValueTraitArguments;

crates/turbo-tasks-macros/Cargo.toml

@@ -15,5 +15,5 @@ convert_case = "0.6.0"
 proc-macro-error = "1.0.4"
 proc-macro2 = { workspace = true }
 quote = { workspace = true }
-syn = { workspace = true, features = ["full", "extra-traits"] }
+syn = { workspace = true, features = ["full", "extra-traits", "visit-mut"] }
 turbo-tasks-macros-shared = { workspace = true }

crates/turbo-tasks-macros/src/generic_type_macro.rs

@@ -0,0 +1,148 @@
+use proc_macro::TokenStream;
+use quote::quote;
+use syn::{parse_macro_input, spanned::Spanned, visit_mut::VisitMut, GenericParam, Lifetime, Type};
+use turbo_tasks_macros_shared::{get_type_ident, GenericTypeInput};
+
+use crate::value_macro::value_type_and_register;
+
+pub fn generic_type(input: TokenStream) -> TokenStream {
+ let input = parse_macro_input!(input as GenericTypeInput);
+
+ for param in &input.generics.params {
+ match param {
+ syn::GenericParam::Type(ty) => {
+ if ty.ident == "Vc" {
+ ty.span()
+ .unwrap()
+ .error("Vc is a reserved name in generic_type")
+ .emit();
+ }
+ }
+ syn::GenericParam::Lifetime(lt) => {
+ lt.span()
+ .unwrap()
+ .error("lifetime parameters are not supported in generic_type")
+ .emit();
+ }
+ syn::GenericParam::Const(c) => {
+ c.span()
+ .unwrap()
+ .error("const parameters are not supported in generic_type")
+ .emit();
+ }
+ }
+ }
+
+ let (impl_generics, _, where_clause) = input.generics.split_for_impl();
+
+ let repr = replace_generics_with_unit(input.generics.params.iter(), &input.ty);
+
+ let ty = input.ty;
+ let Some(ident) = get_type_ident(&ty) else {
+ return quote! {
+ // An error occurred while parsing the ident.
+ }
+ .into();
+ };
+
+ let mut generics_with_static = input.generics.clone();
+ for param in &mut generics_with_static.params {
+ if let GenericParam::Type(param) = param {
+ param.bounds.push(syn::TypeParamBound::Lifetime(Lifetime {
+ ident: syn::Ident::new("static", param.ident.span()),
+ apostrophe: param.ident.span(),
+ }))
+ }
+ }
+
+ let value_type_and_register = value_type_and_register(
+ &ident,
+ quote! { #ty },
+ Some(&generics_with_static),
+ quote! {
+ turbo_tasks::VcTransparentRead<#ty, #ty, #repr>
+ },
+ quote! {
+ turbo_tasks::VcCellSharedMode<#ty>
+ },
+ quote! {
+ turbo_tasks::ValueType::new_with_any_serialization::<#repr>()
+ },
+ );
+
+ quote! {
+ #value_type_and_register
+
+ impl #impl_generics Vc<#ty> #where_clause {
+ /// Converts this `Vc` to a generic representation.
+ fn to_repr(vc: Self) -> Vc<#repr> {
+ unsafe {
+ turbo_tasks::Vc::from_raw(Vc::into_raw(vc))
+ }
+ }
+
+ /// Converts a generic representation of this `Vc` to the proper `Vc` type.
+ ///
+ /// # Safety
+ ///
+ /// The caller must ensure that the `repr` is a valid representation of this `Vc`.
+ unsafe fn from_repr(vc: Vc<#repr>) -> Self {
+ unsafe {
+ turbo_tasks::Vc::from_raw(Vc::into_raw(vc))
+ }
+ }
+ }
+ }
+ .into()
+}
+
+struct ReplaceGenericsVisitor<'a> {
+ generics: &'a std::collections::HashSet<String>,
+}
+
+impl<'a> VisitMut for ReplaceGenericsVisitor<'a> {
+ fn visit_type_mut(&mut self, node: &mut Type) {
+ if let Type::Path(type_path) = node {
+ if type_path.qself.is_none()
+ && type_path.path.segments.len() == 1
+ && type_path.path.segments[0].arguments.is_none()
+ && self
+ .generics
+ .contains(&type_path.path.segments[0].ident.to_string())
+ {
+ // Replace the whole path with ()
+ *node = syn::parse_quote! { () };
+ return;
+ }
+ }
+
+ syn::visit_mut::visit_type_mut(self, node);
+ }
+}
+
+/// Replaces all instances of `params` generic types in `ty` with the unit type
+/// `()`.
+fn replace_generics_with_unit<'a, P>(params: P, ty: &Type) -> Type
+where
+ P: IntoIterator<Item = &'a GenericParam>,
+{
+ let generics_set: std::collections::HashSet<_> = params
+ .into_iter()
+ .filter_map(|param| {
+ if let GenericParam::Type(type_param) = param {
+ Some(type_param.ident.to_string())
+ } else {
+ None
+ }
+ })
+ .collect();
+
+ let mut new_ty = ty.clone();
+ let mut visitor = ReplaceGenericsVisitor {
+ generics: &generics_set,
+ };
+
+ syn::visit_mut::visit_type_mut(&mut visitor, &mut new_ty);
+
+ new_ty
+}

crates/turbo-tasks-macros/src/lib.rs

@@ -5,6 +5,7 @@
 mod derive;
 mod func;
 mod function_macro;
+mod generic_type_macro;
 mod primitive_macro;
 mod value_impl_macro;
 mod value_macro;
@@ -147,3 +148,28 @@ pub fn value_impl(args: TokenStream, input: TokenStream) -> TokenStream {
 pub fn primitive(input: TokenStream) -> TokenStream {
  primitive_macro::primitive(input)
 }
+
+/// Registers a value type that is generic over the `Vc` it contains.
+///
+/// # Example
+///
+/// ```
+/// use crate::generic_type as __turbo_tasks_internal_generic_type;
+///
+/// __turbo_tasks_internal_generic_type!(<A, B>, GenericType<Vc<A>, Vc<B>>);
+///
+/// // Now you can do the following, for any `A` and `B` value types:
+///
+/// let vc: Vc<GenericType<Vc<u32>, Vc<String>>> = Vc::cell(
+/// GenericType::new(
+/// Vc::cell(42),
+/// Vc::cell("hello".to_string())
+/// )
+/// );
+/// ```
+#[allow_internal_unstable(min_specialization, into_future, trivial_bounds)]
+#[proc_macro_error]
+#[proc_macro]
+pub fn generic_type(input: TokenStream) -> TokenStream {
+ generic_type_macro::generic_type(input)
+}

crates/turbo-tasks-macros/src/primitive_macro.rs

@@ -36,8 +36,9 @@ pub fn primitive(input: TokenStream) -> TokenStream {
  let value_type_and_register = value_type_and_register(
  &ident,
  quote! { #ty },
+ None,
  quote! {
- turbo_tasks::VcTransparentRead<#ty, #ty>
+ turbo_tasks::VcTransparentRead<#ty, #ty, #ty>
  },
  quote! {
  turbo_tasks::VcCellSharedMode<#ty>

crates/turbo-tasks-macros/src/value_impl_macro.rs

@@ -6,7 +6,7 @@ use syn::{
  parse_macro_input, parse_quote,
  punctuated::Punctuated,
  spanned::Spanned,
- Attribute, Error, ExprPath, ImplItem, ImplItemMethod, ItemImpl, Lit, LitStr, Meta,
+ Attribute, Error, ExprPath, Generics, ImplItem, ImplItemMethod, ItemImpl, Lit, LitStr, Meta,
  MetaNameValue, Path, Result, Token, Type,
 };
 use turbo_tasks_macros_shared::{
@@ -158,9 +158,9 @@ pub fn value_impl(args: TokenStream, input: TokenStream) -> TokenStream {
  }
 
  #[doc(hidden)]
- pub(crate) static #native_function_ident: #native_function_ty = #ty::#native_function_ident;
+ pub(crate) static #native_function_ident: #native_function_ty = <#ty>::#native_function_ident;
  #[doc(hidden)]
- pub(crate) static #native_function_id_ident: #native_function_id_ty = #ty::#native_function_id_ident;
+ pub(crate) static #native_function_id_ident: #native_function_id_ty = <#ty>::#native_function_id_ident;
  })
  }
  }
@@ -176,12 +176,15 @@ pub fn value_impl(args: TokenStream, input: TokenStream) -> TokenStream {
 
  fn trait_value_impl(
  ty: &Type,
+ generics: &Generics,
  ty_ident: &Ident,
  trait_path: &Path,
  items: &[ImplItem],
  ) -> TokenStream2 {
  let trait_ident = get_path_ident(trait_path);
 
+ let (impl_generics, _, where_clause) = generics.split_for_impl();
+
  let register = get_register_trait_methods_ident(&trait_ident, ty_ident);
 
  let mut trait_registers = Vec::new();
@@ -263,7 +266,7 @@ pub fn value_impl(args: TokenStream, input: TokenStream) -> TokenStream {
 
  #[doc(hidden)]
  // #[turbo_tasks::async_trait]
- impl #inline_extension_trait_ident for #ty {
+ impl #impl_generics #inline_extension_trait_ident for #ty #where_clause  {
  #[allow(declare_interior_mutable_const)]
  #[doc(hidden)]
  const #native_function_ident: #native_function_ty = #native_function_def;
@@ -300,9 +303,9 @@ pub fn value_impl(args: TokenStream, input: TokenStream) -> TokenStream {
  // NOTE(alexkirsz) We can't have a general `turbo_tasks::Upcast<Box<dyn Trait>> for T where T: Trait` because
  // rustc complains: error[E0210]: type parameter `T` must be covered by another type when it appears before
  // the first local type (`dyn Trait`).
- unsafe impl turbo_tasks::Upcast<Box<dyn #trait_path>> for #ty {}
+ unsafe impl #impl_generics turbo_tasks::Upcast<Box<dyn #trait_path>> for #ty #where_clause {}
 
- impl #trait_path for #ty {
+ impl #impl_generics #trait_path for #ty #where_clause {
  #(#trait_functions)*
  }
 
@@ -324,8 +327,13 @@ pub fn value_impl(args: TokenStream, input: TokenStream) -> TokenStream {
 
  match &item.trait_ {
  None => inherent_value_impl(&item.self_ty, &ty_ident, &item.items).into(),
- Some((_, trait_path, _)) => {
- trait_value_impl(&item.self_ty, &ty_ident, trait_path, &item.items).into()
- }
+ Some((_, trait_path, _)) => trait_value_impl(
+ &item.self_ty,
+ &item.generics,
+ &ty_ident,
+ trait_path,
+ &item.items,
+ )
+ .into(),
  }
 }