Allow cast of fat pointers to thin pointers (rust-lang#99)

Also:
* Correctly cast before dereferencing a thin pointer to a flexible
  struct
* Correct search for trait type in adt (ignore primitive types and
  pointers)
* Correct the choice between thin and fat pointers in codegen_fat_ptr
  and codegen_ty_ref (and rewrite to use mir pointer metadata)

Co-authored-by: Mark R. Tuttle <mrtuttle@amazon.com>

Author: 2 people

Diff for: compiler/rustc_codegen_llvm/src/gotoc/place.rs

@@ -267,7 +267,12 @@ impl<'tcx> GotocCtx<'tcx> {
                         // should be the ADT itself. So we need the `.dereference()` here.
                         // Note that this causes problems in `codegen_rvalue_ref()`.
                         // See the comment there for more details.
-                        inner_goto_expr.member("data", &self.symbol_table).dereference()
+                        inner_goto_expr
+                            .member("data", &self.symbol_table)
+                            // In the case of a vtable fat pointer, this data member is a void pointer,
+                            // so ensure the pointer has the correct type before dereferencing it.
+                            .cast_to(self.codegen_ty(inner_mir_typ).to_pointer())
+                            .dereference()
                     }
                     _ => inner_goto_expr.dereference(),
                 };

Diff for: compiler/rustc_codegen_llvm/src/gotoc/rvalue.rs

@@ -4,7 +4,7 @@ use super::cbmc::goto_program::{BuiltinFn, Expr, Location, Stmt, Symbol, Type};
 use super::cbmc::utils::aggr_name;
 use super::cbmc::MachineModel;
 use super::metadata::*;
-use super::typ::pointee_type;
+use super::typ::{is_pointer, pointee_type};
 use super::utils::{dynamic_fat_ptr, slice_fat_ptr};
 use crate::btree_string_map;
 use rustc_middle::mir::{AggregateKind, BinOp, CastKind, NullOp, Operand, Place, Rvalue, UnOp};
@@ -530,6 +530,19 @@ impl<'tcx> GotocCtx<'tcx> {
         )
     }
 
+    pub fn codegen_fat_ptr_to_thin_ptr_cast(
+        &mut self,
+        src: &Operand<'tcx>,
+        dst_t: Ty<'tcx>,
+    ) -> Expr {
+        debug!("codegen_fat_ptr_to_thin_ptr_cast |{:?}| |{:?}|", src, dst_t);
+        let src_goto_expr = self.codegen_operand(src);
+        let dst_goto_typ = self.codegen_ty(dst_t);
+        // In a vtable fat pointer, the data member is a void pointer,
+        // so ensure the pointer has the correct type before dereferencing it.
+        src_goto_expr.member("data", &self.symbol_table).cast_to(dst_goto_typ)
+    }
+
     fn codegen_misc_cast(&mut self, src: &Operand<'tcx>, dst_t: Ty<'tcx>) -> Expr {
         let src_t = self.operand_ty(src);
         debug!(
@@ -558,6 +571,10 @@ impl<'tcx> GotocCtx<'tcx> {
             return self.codegen_fat_ptr_to_fat_ptr_cast(src, dst_t);
         }
 
+        if self.is_ref_of_unsized(src_t) && self.is_ref_of_sized(dst_t) {
+            return self.codegen_fat_ptr_to_thin_ptr_cast(src, dst_t);
+        }
+
         // pointer casting. from a pointer / reference to another pointer / reference
         // notice that if fat pointer is involved, it cannot be the destination, which is t.
         match dst_t.kind() {
@@ -949,8 +966,8 @@ impl<'tcx> GotocCtx<'tcx> {
         if let Some((concrete_type, trait_type)) =
             self.nested_pair_of_concrete_and_trait_types(src_pointee_type, dst_pointee_type)
         {
-            let dst_goto_type = self.codegen_ty(dst_mir_type);
             let dst_goto_expr = src_goto_expr.cast_to(Type::void_pointer());
+            let dst_goto_type = self.codegen_ty(dst_mir_type);
             let vtable = self.codegen_vtable(concrete_type, trait_type);
             let vtable_expr = vtable.address_of();
             Some(dynamic_fat_ptr(dst_goto_type, dst_goto_expr, vtable_expr, &self.symbol_table))
@@ -1020,11 +1037,18 @@ impl<'tcx> GotocCtx<'tcx> {
     /// position a concrete type. This function returns the pair (concrete type,
     /// trait type) that we can use to build the vtable for the concrete type
     /// implementation of the trait type.
-    fn nested_pair_of_concrete_and_trait_types(
+    pub fn nested_pair_of_concrete_and_trait_types(
         &self,
         src_mir_type: Ty<'tcx>,
         dst_mir_type: Ty<'tcx>,
     ) -> Option<(Ty<'tcx>, Ty<'tcx>)> {
+        // We are walking an ADT searching for a trait type in this ADT.  We can
+        // terminate a walk down a path when we hit a primitive type or which we hit
+        // a pointer type (that would take us out of this ADT and into another type).
+        if dst_mir_type.is_primitive() || is_pointer(dst_mir_type) {
+            return None;
+        }
+
         match (src_mir_type.kind(), dst_mir_type.kind()) {
             (_, ty::Dynamic(..)) => Some((src_mir_type.clone(), dst_mir_type.clone())),
             (ty::Adt(..), ty::Adt(..)) => {
@@ -1067,7 +1091,7 @@ impl<'tcx> GotocCtx<'tcx> {
             //     Some((src_mir_type.clone(), dst_mir_type.clone()))
             // }
             _ => panic!(
-                "Searching for pairs of concrete and trait types found unexpected types in {:?} and {:?}",
+                "Found unexpected types while searching for pairs of concrete and trait types in {:?} and {:?}",
                 src_mir_type, dst_mir_type
             ),
         }

Diff for: compiler/rustc_codegen_llvm/src/gotoc/typ.rs

@@ -96,6 +96,14 @@ impl<'tcx> GotocCtx<'tcx> {
         }
     }
 
+    /// Is the MIR type a ref of an unsized type (i.e. one represented by a fat pointer?)
+    pub fn is_ref_of_sized(&self, t: &'tcx TyS<'_>) -> bool {
+        match t.kind() {
+            ty::Ref(_, to, _) | ty::RawPtr(ty::TypeAndMut { ty: to, .. }) => !self.is_unsized(to),
+            _ => false,
+        }
+    }
+
     /// Is the MIR type a box of an unsized type (i.e. one represented by a fat pointer?)
     pub fn is_box_of_unsized(&self, t: &'tcx TyS<'_>) -> bool {
         if t.is_box() {
@@ -519,65 +527,61 @@ impl<'tcx> GotocCtx<'tcx> {
         })
     }
 
-    pub fn codegen_fat_ptr(&mut self, t: Ty<'tcx>) -> Type {
-        // implement "fat pointers", which are pointers with metadata.
-        // to my knowledge, there are three kinds of fat pointers:
-        // 1. references to slices
-        // 2. references to trait objects.
-        // 3. references to structs whose last field is a fat pointer
-        match t.kind() {
-            ty::Adt(..) if self.is_unsized(t) => {
-                // https://rust-lang.zulipchat.com/#narrow/stream/182449-t-compiler.2Fhelp
-                let fat_ptr_name = format!("&{}", self.ty_mangled_name(t));
-                self.ensure_struct(&fat_ptr_name, |ctx, _| {
-                    vec![
-                        Type::datatype_component("data", ctx.codegen_ty(t).to_pointer()),
-                        Type::datatype_component("len", Type::size_t()),
-                    ]
-                })
-            }
-            ty::Slice(e) => {
-                // a slice &[t] is translated to
-                // struct {
-                //   t *data;
-                //   usize len;
-                // }
-                // c.f. core::ptr::FatPtr<T>
-                let slice_name = self.ty_mangled_name(t);
-                self.ensure_struct(&slice_name, |ctx, _| {
-                    vec![
-                        Type::datatype_component("data", ctx.codegen_ty(e).to_pointer()),
-                        Type::datatype_component("len", Type::size_t()),
-                    ]
-                })
-            }
-            ty::Str => self.ensure_struct("str", |_, _| {
+    pub fn codegen_fat_ptr(&mut self, mir_type: Ty<'tcx>) -> Type {
+        assert!(
+            !self.use_thin_pointer(mir_type),
+            "Generating a fat pointer for a type requiring a thin pointer: {:?}",
+            mir_type.kind()
+        );
+        if self.use_slice_fat_pointer(mir_type) {
+            let pointer_name = match mir_type.kind() {
+                ty::Slice(..) => self.ty_mangled_name(mir_type),
+                ty::Str => "str".to_string(),
+                ty::Adt(..) => format!("&{}", self.ty_mangled_name(mir_type)),
+                kind => unreachable!("Generating a slice fat pointer to {:?}", kind),
+            };
+            let element_type = match mir_type.kind() {
+                ty::Slice(elt_type) => self.codegen_ty(elt_type),
+                ty::Str => Type::c_char(),
+                // For adt, see https://rust-lang.zulipchat.com/#narrow/stream/182449-t-compiler.2Fhelp
+                ty::Adt(..) => self.codegen_ty(mir_type),
+                kind => unreachable!("Generating a slice fat pointer to {:?}", kind),
+            };
+            self.ensure_struct(&pointer_name, |_, _| {
                 vec![
-                    Type::datatype_component("data", Type::c_char().to_pointer()),
+                    Type::datatype_component("data", element_type.to_pointer()),
                     Type::datatype_component("len", Type::size_t()),
                 ]
-            }),
-            ty::Dynamic(binder, _region) => {
-                debug!("type codegen for dynamic with binder {:?} type {:?}", binder, t);
-                self.codegen_trait_vtable_type(t);
-                self.codegen_trait_fat_ptr_type(t)
-            }
-
-            _ => unreachable!(),
+            })
+        } else if self.use_vtable_fat_pointer(mir_type) {
+            let (_, trait_type) =
+                self.nested_pair_of_concrete_and_trait_types(mir_type, mir_type).unwrap();
+            self.codegen_trait_vtable_type(trait_type);
+            self.codegen_trait_fat_ptr_type(trait_type)
+        } else {
+            unreachable!(
+                "A pointer is either a thin pointer, slice fat pointer, or vtable fat pointer."
+            );
         }
     }
 
-    pub fn codegen_ty_ref(&mut self, t: Ty<'tcx>) -> Type {
-        // Normalize t to remove projection and opaque types
-        let t = self.tcx.normalize_erasing_regions(ty::ParamEnv::reveal_all(), t);
+    pub fn codegen_ty_ref(&mut self, pointee_type: Ty<'tcx>) -> Type {
+        // Normalize pointee_type to remove projection and opaque types
+        let pointee_type =
+            self.tcx.normalize_erasing_regions(ty::ParamEnv::reveal_all(), pointee_type);
 
-        match t.kind() {
-            // These types go to fat pointers
-            ty::Dynamic(..) | ty::Slice(_) | ty::Str => self.codegen_fat_ptr(t),
-            ty::Adt(..) if self.is_unsized(t) => self.codegen_fat_ptr(t),
+        if !self.use_thin_pointer(pointee_type) {
+            return self.codegen_fat_ptr(pointee_type);
+        }
+
+        match pointee_type.kind() {
+            ty::Dynamic(..) | ty::Slice(_) | ty::Str => {
+                unreachable!("Should have generated a fat pointer")
+            }
             ty::Projection(_) | ty::Opaque(..) => {
                 unreachable!("Should have been removed by normalization")
             }
+
             // We have a "thin pointer", which is just a pointer
             ty::Adt(..)
             | ty::Array(..)
@@ -590,22 +594,22 @@ impl<'tcx> GotocCtx<'tcx> {
             | ty::RawPtr(_)
             | ty::Ref(..)
             | ty::Tuple(_)
-            | ty::Uint(_) => self.codegen_ty(t).to_pointer(),
+            | ty::Uint(_) => self.codegen_ty(pointee_type).to_pointer(),
 
             // These types were blocking firecracker. Doing the default thing to unblock.
             // TODO, determine if this is the right course of action
-            ty::FnDef(_, _) | ty::Never => self.codegen_ty(t).to_pointer(),
+            ty::FnDef(_, _) | ty::Never => self.codegen_ty(pointee_type).to_pointer(),
 
             // These types have no regression tests for them.
             // For soundess, hold off on generating them till we have test-cases.
-            ty::Bound(_, _) => todo!("{:?} {:?}", t, t.kind()),
-            ty::Error(_) => todo!("{:?} {:?}", t, t.kind()),
-            ty::FnPtr(_) => todo!("{:?} {:?}", t, t.kind()),
-            ty::Generator(_, _, _) => todo!("{:?} {:?}", t, t.kind()),
-            ty::GeneratorWitness(_) => todo!("{:?} {:?}", t, t.kind()),
-            ty::Infer(_) => todo!("{:?} {:?}", t, t.kind()),
-            ty::Param(_) => todo!("{:?} {:?}", t, t.kind()),
-            ty::Placeholder(_) => todo!("{:?} {:?}", t, t.kind()),
+            ty::Bound(_, _) => todo!("{:?} {:?}", pointee_type, pointee_type.kind()),
+            ty::Error(_) => todo!("{:?} {:?}", pointee_type, pointee_type.kind()),
+            ty::FnPtr(_) => todo!("{:?} {:?}", pointee_type, pointee_type.kind()),
+            ty::Generator(_, _, _) => todo!("{:?} {:?}", pointee_type, pointee_type.kind()),
+            ty::GeneratorWitness(_) => todo!("{:?} {:?}", pointee_type, pointee_type.kind()),
+            ty::Infer(_) => todo!("{:?} {:?}", pointee_type, pointee_type.kind()),
+            ty::Param(_) => todo!("{:?} {:?}", pointee_type, pointee_type.kind()),
+            ty::Placeholder(_) => todo!("{:?} {:?}", pointee_type, pointee_type.kind()),
         }
     }
 
@@ -1006,6 +1010,11 @@ impl<'tcx> GotocCtx<'tcx> {
     }
 }
 
+/// The mir type is a mir pointer type.
+pub fn is_pointer(mir_type: Ty<'tcx>) -> bool {
+    return matches!(mir_type.kind(), ty::Ref(..) | ty::RawPtr(..));
+}
+
 /// Extract from a mir pointer type the mir type of the value to which the
 /// pointer points.
 pub fn pointee_type(pointer_type: Ty<'tcx>) -> Option<Ty<'tcx>> {
@@ -1019,7 +1028,9 @@ pub fn pointee_type(pointer_type: Ty<'tcx>) -> Option<Ty<'tcx>> {
 impl<'tcx> GotocCtx<'tcx> {
     /// A pointer to the mir type should be a thin pointer.
     pub fn use_thin_pointer(&self, mir_type: Ty<'tcx>) -> bool {
-        return mir_type.ptr_metadata_ty(self.tcx) == self.tcx.types.unit;
+        // ptr_metadata_ty is not defined on all types, the projection of an associated type
+        return !self.is_unsized(mir_type)
+            || mir_type.ptr_metadata_ty(self.tcx) == self.tcx.types.unit;
     }
     /// A pointer to the mir type should be a slice fat pointer.
     pub fn use_slice_fat_pointer(&self, mir_type: Ty<'tcx>) -> bool {

Diff for: rust-tests/cbmc-reg/SizeAndAlignOfDst/main_fail.rs

@@ -6,7 +6,6 @@
 // This test still fails with a final coercion error for
 // DummySubscriber to dyn Subscriber.
 
-#![feature(layout_for_ptr)]
 use std::mem;
 use std::sync::Arc;
 use std::sync::Mutex;
@@ -30,13 +29,5 @@ impl Subscriber for DummySubscriber {
 }
 
 fn main() {
-    let v = unsafe { mem::size_of_val_raw(&5i32) };
-    assert!(v == 4);
-
-    let x: [u8; 13] = [0; 13];
-    let y: &[u8] = &x;
-    let v = unsafe { mem::size_of_val_raw(y) };
-    assert!(v == 13);
-
     let s: Arc<Mutex<dyn Subscriber>> = Arc::new(Mutex::new(DummySubscriber::new()));
 }