Complete Dart slice borrowing (#437)

core/src/hir/lifetimes.rs

@@ -78,6 +78,11 @@ impl LifetimeEnv {
  (0..self.num_lifetimes()).map(Lifetime::new)
  }
 
+ /// Get the bounds for a named lifetime (none for unnamed lifetimes)
+ pub(super) fn get_bounds(&self, named_lifetime: Lifetime) -> Option<&BoundedLifetime> {
+ self.nodes.get(named_lifetime.0)
+ }
+
  /// Returns a new [`LifetimeEnv`].
  pub(super) fn new(
  nodes: SmallVec<[BoundedLifetime; INLINE_NUM_LIFETIMES]>,
@@ -386,6 +391,15 @@ impl<'def, 'tcx> LinkedLifetimes<'def, 'tcx> {
  }
  }
 
+ /// Takes a lifetime at the def site and produces one at the use site
+ pub fn def_to_use(&self, def_lt: Lifetime) -> MaybeStatic<Lifetime> {
+ *self
+ .lifetimes
+ .as_slice()
+ .get(def_lt.0)
+ .expect("All def site lifetimes must be used!")
+ }
+
  /// The lifetime env at the def site. Def lifetimes should be resolved
  /// against this.
  pub fn def_env(&self) -> &'tcx LifetimeEnv {

core/src/hir/methods.rs

@@ -130,6 +130,20 @@ impl ReturnType {
 
  set
  }
+
+ pub fn with_contained_types(&self, mut f: impl FnMut(&OutType)) {
+ match self {
+ Self::Infallible(SuccessType::OutType(o))
+ | Self::Nullable(SuccessType::OutType(o))
+ | Self::Fallible(SuccessType::OutType(o), None) => f(o),
+ Self::Fallible(SuccessType::OutType(o), Some(o2)) => {
+ f(o);
+ f(o2)
+ }
+ Self::Fallible(_, Some(o)) => f(o),
+ _ => (),
+ }
+ }
 }
 
 impl ParamSelf {

core/src/hir/methods/borrowing_param.rs

@@ -3,12 +3,88 @@
 //! This is useful for backends which wish to figure out the borrowing relationships between parameters
 //! and return values,
 //! and then delegate how lifetimes get mapped to fields to the codegen for those types respectively.
+//!
+//!
+//! # Typical usage
+//!
+//! In managed languages, to ensure the GC doesn't prematurely clean up the borrowed-from object,
+//! we use the technique of stashing a borrowed-from object as a field of a borrowed-to object.
+//! This is called a "lifetime edge" (since it's an edge in the GC graph). An "edge array" is typically
+//! a stash of all the edges on an object, or all the edges corresponding to a particular lifetime.
+//!
+//! This stashing is primarily driven by method codegen, which is where actual borrow relationships can be set up.
+//!
+//! Typically, codegen for a method should instantiate a [`BorrowedParamVisitor`] and uses it to [`BorrowedParamVisitor::visit_param()`] each input parameter to determine if it needs to be linked into an edge.
+//! Whilst doing so, it may return useful [`ParamBorrowInfo`] which provides additional information on handling structs and slices. More on structs later.
+//! At the end of the visiting, [`BorrowedLifetimeInfo::borrow_map()`] can be called to get a list of relevant input edges: each lifetime that participates in borrowing,
+//! and a list of parameter names that it borrows from.
+//!
+//! This visitor will automatically handle transitive lifetime relationships as well.
+//!
+//! These edges should be put together into "edge arrays" which then get passed down to output types, handling transitive lifetimes as necessary.
+//!
+//! ## Method codegen for slices
+//!
+//! Slices are strange: in managed languages a host slice will not be a Rust slice, unlike other borrowed host values (a borrowed host opaque is just a borrowed Rust
+//! opaque). We need to convert these across the boundary by allocating a temporary arena or something.
+//!
+//! If the slice doesn't participate in borrowing, this is easy: just allocate a temporary arena. However, if not, we need to allocate an arena with a finalizer (or something similar)
+//! and add that arena to the edge instead. [`LifetimeEdgeKind`] has special handling for this.
+//!
+//! Whether or not the slice participates in borrowing is found out from the [`ParamBorrowInfo`] returned by [`BorrowedParamVisitor::visit_param()`].
+//!
+//! # Method codegen for struct params
+//!
+//! Structs can contain many lifetimes and have relationships between them. Generally, a Diplomat struct is a "bag of stuff"; it is converted to a host value that is structlike, with fields
+//! individually converted.
+//!
+//! Diplomat enforces that struct lifetime bounds never imply additional bounds on methods during HIR validation, so the relationships between struct
+//! lifetimes are not relevant for code here (and as such you should hopefully never need to call [`LifetimeEnv::all_longer_lifetimes()`])
+//! for a struct env).
+//!
+//! The borrowing technique in this module allows a lot of things to be delegated to structs. As will be explained below, structs will have:
+//!
+//! - A `fields_for_lifetime_foo()` set of methods that returns all non-slice fields corresponding to a lifetime
+//! - "append array" outparams for stashing edges when converting from host-to-Rust
+//! - Some way of passing down edge arrays to individual borrowed fields when constructing Rust-to-host
+//!
+//! In methods, when constructing the edge arrays, `fields_for_lifetime_foo()` for every borrowed param can be appended to each
+//! one whenever [`LifetimeEdgeKind::StructLifetime`] asks you to do so. The code needs to handle lifetime
+//! transitivity, since the struct will not be doing so. Fortunately the edges produced by [`BorrowedParamVisitor`] already do so.
+//!
+//! Then, when converting structs host-to-Rust, every edge array relevant to a struct lifetime should be passed in for an append array. Append arrays
+//! are nested arrays: they are an array of edge arrays. The struct will append further things to the edge array.
+//!
+//! Finally, when converting Rust-to-host, if producing a struct, make sure to pass in all the edge arrays.
+//!
+//! # Struct codegen
+//!
+//! At a high level, struct codegen needs to deal with lifetimes in three places:
+//!
+//! - A `fields_for_lifetime_foo()` set of methods that returns all non-slice fields corresponding to a lifetime
+//! - "append array" outparams for stashing edges when converting from host-to-Rust
+//! - Some way of passing down edge arrays to individual borrowed fields when constructing Rust-to-host
+//!
+//! The backend may choose to handle just slices via append arrays, or handle all borrowing there, in which case `fields_for_lifetime_foo()` isn't necessary.
+//!
+//! `fields_for_lifetime_foo()` should return an iterator/list/etc corresponding to each field that *directly* borrows from lifetime foo.
+//! This may include calling `fields_for_lifetime_bar()` on fields that are themselves structs. As mentioned previously, lifetime relationships
+//! are handled by methods and don't need to be dealt with here. There are no helpers for doing this but it's just a matter of
+//! filtering for fields using the lifetime, except for handling nested structs ([`StructBorrowInfo::compute_for_struct_field()`])
+//!
+//! Append arrays are similarly straightforward: for each lifetime on the struct, the host-to-Rust constructor should accept a list of edge arrays. For slice fields,
+//! if the appropriate list is nonempty, the slice's arena edge should be appended to all of the edge arrays in it. These arrays should also be passed down to struct fields
+//! appropriately: [`StructBorrowInfo::compute_for_struct_field()`] is super helpful for getting a list edges something should get passed down to.
+//!
+//! Finally, when converting Rust-to-host, the relevant lifetime edges should be proxied down to the final host type constructors who can stash them wherever needed.
+//! This is one again a matter of filtering fields by the lifetimes they use, and for nested structs you can use [`StructBorrowInfo::compute_for_struct_field()`].
 
 use std::collections::{BTreeMap, BTreeSet};
 
-use crate::hir::{self, Method, TypeContext};
+use crate::hir::{self, Method, StructDef, TyPosition, TypeContext};
 
 use crate::hir::lifetimes::{Lifetime, LifetimeEnv, MaybeStatic};
+use crate::hir::ty_position::StructPathLike;
 
 /// A visitor for processing method parameters/returns and understanding their borrowing relationships, shallowly.
 ///
@@ -19,8 +95,6 @@ use crate::hir::lifetimes::{Lifetime, LifetimeEnv, MaybeStatic};
 /// Obtain from [`Method::borrowing_param_visitor()`].
 pub struct BorrowingParamVisitor<'tcx> {
  tcx: &'tcx TypeContext,
- #[allow(unused)] // to be used later
- method: &'tcx Method,
  used_method_lifetimes: BTreeSet<Lifetime>,
  borrow_map: BTreeMap<Lifetime, BorrowedLifetimeInfo<'tcx>>,
 }
@@ -78,7 +152,6 @@ impl<'tcx> BorrowingParamVisitor<'tcx> {
  .collect();
  BorrowingParamVisitor {
  tcx,
- method,
  used_method_lifetimes,
  borrow_map,
  }
@@ -94,23 +167,28 @@ impl<'tcx> BorrowingParamVisitor<'tcx> {
  self.borrow_map
  }
 
- /// Processes a parameter, adding it to the borrow_map for any lifetimes it references. Returns true if the lifetime is referenced.
+ /// Processes a parameter, adding it to the borrow_map for any lifetimes it references. Returns further information about the type of borrow.
  ///
  /// This basically boils down to: For each lifetime that is actually relevant to borrowing in this method, check if that
  /// lifetime or lifetimes longer than it are used by this parameter. In other words, check if
  /// it is possible for data in the return type with this lifetime to have been borrowed from this parameter.
  /// If so, add code that will yield the ownership-relevant parts of this object to incoming_edges for that lifetime.
- pub fn visit_param(&mut self, ty: &hir::Type, param_name: &str) -> bool {
+ pub fn visit_param(&mut self, ty: &hir::Type, param_name: &str) -> ParamBorrowInfo<'tcx> {
+ let mut is_borrowed = false;
  if self.used_method_lifetimes.is_empty() {
- return false;
+ if let hir::Type::Slice(..) = *ty {
+ return ParamBorrowInfo::TemporarySlice;
+ } else {
+ return ParamBorrowInfo::NotBorrowed;
+ }
  }
- let mut edges_pushed = false;
 
  // Structs have special handling: structs are purely Dart-side, so if you borrow
  // from a struct, you really are borrowing from the internal fields.
  if let hir::Type::Struct(s) = ty {
+ let mut borrowed_struct_lifetime_map = BTreeMap::<Lifetime, BTreeSet<Lifetime>>::new();
  let link = s.link_lifetimes(self.tcx);
- for method_lifetime in self.borrow_map.values_mut() {
+ for (method_lifetime, method_lifetime_info) in &mut self.borrow_map {
  // Note that ty.lifetimes()/s.lifetimes() is lifetimes
  // in the *use* context, i.e. lifetimes on the Type that reference the
  // indices of the method's lifetime arrays. Their *order* references
@@ -124,19 +202,33 @@ impl<'tcx> BorrowingParamVisitor<'tcx> {
  // This is a struct so lifetimes_def_only() is fine to call
  for (use_lt, def_lt) in link.lifetimes_def_only() {
  if let MaybeStatic::NonStatic(use_lt) = use_lt {
- if method_lifetime.all_longer_lifetimes.contains(&use_lt) {
+ if method_lifetime_info.all_longer_lifetimes.contains(&use_lt) {
  let edge = LifetimeEdge {
  param_name: param_name.into(),
  kind: LifetimeEdgeKind::StructLifetime(link.def_env(), def_lt),
  };
- method_lifetime.incoming_edges.push(edge);
- edges_pushed = true;
+ method_lifetime_info.incoming_edges.push(edge);
+
+ is_borrowed = true;
+
+ borrowed_struct_lifetime_map
+ .entry(def_lt)
+ .or_default()
+ .insert(*method_lifetime);
  // Do *not* break the inner loop here: even if we found *one* matching lifetime
  // in this struct that may not be all of them, there may be some other fields that are borrowed
  }
  }
  }
  }
+ if is_borrowed {
+ ParamBorrowInfo::Struct(StructBorrowInfo {
+ env: link.def_env(),
+ borrowed_struct_lifetime_map,
+ })
+ } else {
+ ParamBorrowInfo::NotBorrowed
+ }
  } else {
  for method_lifetime in self.borrow_map.values_mut() {
  for lt in ty.lifetimes() {
@@ -154,16 +246,99 @@ impl<'tcx> BorrowingParamVisitor<'tcx> {
  };
 
  method_lifetime.incoming_edges.push(edge);
- edges_pushed = true;
+ is_borrowed = true;
  // Break the inner loop: we've already determined this
  break;
  }
  }
  }
  }
+ match (is_borrowed, ty) {
+ (true, &hir::Type::Slice(..)) => ParamBorrowInfo::BorrowedSlice,
+ (false, &hir::Type::Slice(..)) => ParamBorrowInfo::TemporarySlice,
+ (false, _) => ParamBorrowInfo::NotBorrowed,
+ (true, _) => ParamBorrowInfo::BorrowedOpaque,
+ }
  }
+ }
+}
 
- // return true if the number of edges changed
- edges_pushed
+/// Information relevant to borrowing for producing conversions
+#[derive(Clone, Debug)]
+#[non_exhaustive]
+pub enum ParamBorrowInfo<'tcx> {
+ /// No borrowing constraints. This means the parameter
+ /// is not borrowed by the output and also does not need temporary borrows
+ NotBorrowed,
+ /// A slice that is not borrowed by the output (but will still need temporary allocation)
+ TemporarySlice,
+ /// A slice that is borrowed by the output
+ BorrowedSlice,
+ /// A struct parameter that is borrowed by the output
+ Struct(StructBorrowInfo<'tcx>),
+ /// An opaque type that is borrowed
+ BorrowedOpaque,
+}
+
+/// Information about the lifetimes of a struct parameter that are borrowed by a method output or by a wrapping struct
+#[derive(Clone, Debug)]
+#[non_exhaustive]
+pub struct StructBorrowInfo<'tcx> {
+ /// This is the struct's lifetime environment
+ pub env: &'tcx LifetimeEnv,
+ /// A map from (borrow-relevant) struct lifetimes to lifetimes in the method (or wrapping struct) that may flow from it
+ pub borrowed_struct_lifetime_map: BTreeMap<Lifetime, BTreeSet<Lifetime>>,
+}
+
+impl<'tcx> StructBorrowInfo<'tcx> {
+ /// Get borrowing info for a struct field, if it does indeed borrow
+ ///
+ /// The lifetime map produced here does not handle lifetime dependencies: the expectation is that the struct
+ /// machinery generated by this will be called by method code that handles these dependencies. We try to handle
+ /// lifetime dependencies in ONE place.
+ pub fn compute_for_struct_field<P: TyPosition>(
+ struc: &StructDef<P>,
+ field: &P::StructPath,
+ tcx: &'tcx TypeContext,
+ ) -> Option<Self> {
+ if field.lifetimes().as_slice().is_empty() {
+ return None;
+ }
+
+ let mut borrowed_struct_lifetime_map = BTreeMap::<Lifetime, BTreeSet<Lifetime>>::new();
+
+ let link = field.link_lifetimes(tcx);
+
+ for outer_lt in struc.lifetimes.all_lifetimes() {
+ // Note that field.lifetimes()
+ // in the *use* context, i.e. lifetimes on the Type that reference the
+ // indices of the outer struct's lifetime arrays. Their *order* references
+ // the indices of the underlying struct def. We need to link the two,
+ // since the _fields_for_lifetime_foo() methods are named after
+ // the *def* context lifetime.
+ //
+ // This is a struct so lifetimes_def_only() is fine to call
+ for (use_lt, def_lt) in link.lifetimes_def_only() {
+ if let MaybeStatic::NonStatic(use_lt) = use_lt {
+ // We do *not* need to transitively check for longer lifetimes here:
+ //
+ if outer_lt == use_lt {
+ borrowed_struct_lifetime_map
+ .entry(def_lt)
+ .or_default()
+ .insert(outer_lt);
+ }
+ }
+ }
+ }
+ if borrowed_struct_lifetime_map.is_empty() {
+ // if the inner struct is only statics
+ None
+ } else {
+ Some(StructBorrowInfo {
+ env: link.def_env(),
+ borrowed_struct_lifetime_map,
+ })
+ }
  }
 }

core/src/hir/paths.rs

@@ -124,18 +124,6 @@ impl ReturnableStructPath {
  Self::OutStruct(p) => &p.lifetimes,
  }
  }
-
- /// Get a map of lifetimes used on this path to lifetimes as named in the def site. See [`LinkedLifetimes`]
- /// for more information.
- pub fn link_lifetimes<'def, 'tcx>(
- &'def self,
- tcx: &'tcx TypeContext,
- ) -> LinkedLifetimes<'def, 'tcx> {
- match self {
- Self::Struct(p) => p.link_lifetimes(tcx),
- Self::OutStruct(p) => p.link_lifetimes(tcx),
- }
- }
 }
 
 impl<P: TyPosition> StructPath<P> {
@@ -149,17 +137,6 @@ impl StructPath {
  pub fn resolve<'tcx>(&self, tcx: &'tcx TypeContext) -> &'tcx StructDef {
  tcx.resolve_struct(self.tcx_id)
  }
-
- /// Get a map of lifetimes used on this path to lifetimes as named in the def site. See [`LinkedLifetimes`]
- /// for more information.
- pub fn link_lifetimes<'def, 'tcx>(
- &'def self,
- tcx: &'tcx TypeContext,
- ) -> LinkedLifetimes<'def, 'tcx> {
- let struc = self.resolve(tcx);
- let env = &struc.lifetimes;
- LinkedLifetimes::new(env, None, &self.lifetimes)
- }
 }
 
 impl OutStructPath {

core/src/hir/snapshots/diplomat_core__hir__type_context__tests__required_implied_bounds.snap

@@ -0,0 +1,11 @@
+---
+source: core/src/hir/type_context.rs
+expression: output
+---
+Lowering error: Opaque::use_foo should explicitly include this lifetime bound from param foo: 'y: 'x (comes from source type's 'b: 'a)
+Lowering error: Opaque::use_foo should explicitly include this lifetime bound from param foo: 'z: 'y (comes from source type's 'c: 'b)
+Lowering error: Opaque::return_foo should explicitly include this lifetime bound from param return type: 'y: 'x (comes from source type's 'b: 'a)
+Lowering error: Opaque::return_foo should explicitly include this lifetime bound from param return type: 'z: 'y (comes from source type's 'c: 'b)
+Lowering error: Opaque::return_result_foo should explicitly include this lifetime bound from param return type: 'y: 'x (comes from source type's 'b: 'a)
+Lowering error: Opaque::return_result_foo should explicitly include this lifetime bound from param return type: 'z: 'y (comes from source type's 'c: 'b)
+