From d45378216b31eab9ee7c7c461ae20bfb29bd20b3 Mon Sep 17 00:00:00 2001 From: Simonas Kazlauskas Date: Wed, 18 Apr 2018 18:38:12 +0300 Subject: [PATCH 1/4] Change align_offset to support different strides MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is necessary if we want to implement `[T]::align_to` and is more useful in general. This implementation effort has begun during the All Hands and represents a month of my futile efforts to do any sort of maths. Luckily, I found the very very nice Chris McDonald (cjm) on IRC who figured out the core formulas for me! All the thanks for existence of this PR go to them! Anyway… Those formulas were mangled by yours truly into the arcane forms you see here to squeeze out the best assembly possible on most of the modern architectures (x86 and ARM were evaluated in practice). I mean, just look at it: *one actual* modulo operation and everything else is just the cheap single cycle ops! Admitedly, the naive solution might be faster in some common scenarios, but this code absolutely butchers the naive solution on the worst case scenario. Alas, the result of this arcane magic also means that the code pretty heavily relies on the preconditions holding true and breaking those preconditions will unleash the UB-est of all UBs! So don’t. --- src/libcore/intrinsics.rs | 30 ++- src/libcore/ptr.rs | 259 ++++++++++++++++++++----- src/libcore/slice/mod.rs | 22 +++ src/librustc/middle/lang_items.rs | 3 + src/librustc_codegen_llvm/intrinsic.rs | 32 ++- src/librustc_typeck/check/intrinsic.rs | 3 +- src/test/run-pass/align-offset-sign.rs | 82 +++++++- 7 files changed, 363 insertions(+), 68 deletions(-) diff --git a/src/libcore/intrinsics.rs b/src/libcore/intrinsics.rs index 5ec6cb6c71054..510a5bb3df7ba 100644 --- a/src/libcore/intrinsics.rs +++ b/src/libcore/intrinsics.rs @@ -1463,15 +1463,26 @@ extern "rust-intrinsic" { /// source as well as std's catch implementation. pub fn try(f: fn(*mut u8), data: *mut u8, local_ptr: *mut u8) -> i32; - /// Computes the byte offset that needs to be applied to `ptr` in order to - /// make it aligned to `align`. - /// If it is not possible to align `ptr`, the implementation returns + #[cfg(stage0)] + /// docs my friends, its friday! + pub fn align_offset(ptr: *const (), align: usize) -> usize; + + /// Computes the offset that needs to be applied to the pointer in order to make it aligned to + /// `align`. + /// + /// If it is not possible to align the pointer, the implementation returns /// `usize::max_value()`. /// - /// There are no guarantees whatsover that offsetting the pointer will not - /// overflow or go beyond the allocation that `ptr` points into. - /// It is up to the caller to ensure that the returned offset is correct - /// in all terms other than alignment. + /// The offset is expressed in number of `T` elements, and not bytes. The value returned can be + /// used with the `offset` or `offset_to` methods. + /// + /// There are no guarantees whatsover that offsetting the pointer will not overflow or go + /// beyond the allocation that the pointer points into. It is up to the caller to ensure that + /// the returned offset is correct in all terms other than alignment. + /// + /// # Unsafety + /// + /// `align` must be a power-of-two. /// /// # Examples /// @@ -1485,7 +1496,7 @@ extern "rust-intrinsic" { /// # unsafe { /// let x = [5u8, 6u8, 7u8, 8u8, 9u8]; /// let ptr = &x[n] as *const u8; - /// let offset = align_offset(ptr as *const (), align_of::()); + /// let offset = align_offset(ptr, align_of::()); /// if offset < x.len() - n - 1 { /// let u16_ptr = ptr.offset(offset as isize) as *const u16; /// assert_ne!(*u16_ptr, 500); @@ -1495,7 +1506,8 @@ extern "rust-intrinsic" { /// } /// # } } /// ``` - pub fn align_offset(ptr: *const (), align: usize) -> usize; + #[cfg(not(stage0))] + pub fn align_offset(ptr: *const T, align: usize) -> usize; /// Emits a `!nontemporal` store according to LLVM (see their docs). /// Probably will never become stable. diff --git a/src/libcore/ptr.rs b/src/libcore/ptr.rs index 63bcc02402015..a762a8a6f924d 100644 --- a/src/libcore/ptr.rs +++ b/src/libcore/ptr.rs @@ -1433,15 +1433,22 @@ impl *const T { copy_nonoverlapping(self, dest, count) } - /// Computes the byte offset that needs to be applied in order to - /// make the pointer aligned to `align`. + /// Computes the offset that needs to be applied to the pointer in order to make it aligned to + /// `align`. + /// /// If it is not possible to align the pointer, the implementation returns /// `usize::max_value()`. /// - /// There are no guarantees whatsover that offsetting the pointer will not - /// overflow or go beyond the allocation that the pointer points into. - /// It is up to the caller to ensure that the returned offset is correct - /// in all terms other than alignment. + /// The offset is expressed in number of `T` elements, and not bytes. The value returned can be + /// used with the `offset` or `offset_to` methods. + /// + /// There are no guarantees whatsover that offsetting the pointer will not overflow or go + /// beyond the allocation that the pointer points into. It is up to the caller to ensure that + /// the returned offset is correct in all terms other than alignment. + /// + /// # Panics + /// + /// The function panics if `align` is not a power-of-two. /// /// # Examples /// @@ -1465,13 +1472,30 @@ impl *const T { /// # } } /// ``` #[unstable(feature = "align_offset", issue = "44488")] - pub fn align_offset(self, align: usize) -> usize { + #[cfg(not(stage0))] + pub fn align_offset(self, align: usize) -> usize where T: Sized { + if !align.is_power_of_two() { + panic!("align_offset: align is not a power-of-two"); + } unsafe { - intrinsics::align_offset(self as *const _, align) + intrinsics::align_offset(self, align) + } + } + + /// definitely docs. + #[unstable(feature = "align_offset", issue = "44488")] + #[cfg(stage0)] + pub fn align_offset(self, align: usize) -> usize where T: Sized { + if !align.is_power_of_two() { + panic!("align_offset: align is not a power-of-two"); + } + unsafe { + intrinsics::align_offset(self as *const (), align) } } } + #[lang = "mut_ptr"] impl *mut T { /// Returns `true` if the pointer is null. @@ -1804,44 +1828,6 @@ impl *mut T { (self as *const T).wrapping_offset_from(origin) } - /// Computes the byte offset that needs to be applied in order to - /// make the pointer aligned to `align`. - /// If it is not possible to align the pointer, the implementation returns - /// `usize::max_value()`. - /// - /// There are no guarantees whatsover that offsetting the pointer will not - /// overflow or go beyond the allocation that the pointer points into. - /// It is up to the caller to ensure that the returned offset is correct - /// in all terms other than alignment. - /// - /// # Examples - /// - /// Accessing adjacent `u8` as `u16` - /// - /// ``` - /// # #![feature(align_offset)] - /// # fn foo(n: usize) { - /// # use std::mem::align_of; - /// # unsafe { - /// let x = [5u8, 6u8, 7u8, 8u8, 9u8]; - /// let ptr = &x[n] as *const u8; - /// let offset = ptr.align_offset(align_of::()); - /// if offset < x.len() - n - 1 { - /// let u16_ptr = ptr.offset(offset as isize) as *const u16; - /// assert_ne!(*u16_ptr, 500); - /// } else { - /// // while the pointer can be aligned via `offset`, it would point - /// // outside the allocation - /// } - /// # } } - /// ``` - #[unstable(feature = "align_offset", issue = "44488")] - pub fn align_offset(self, align: usize) -> usize { - unsafe { - intrinsics::align_offset(self as *const _, align) - } - } - /// Calculates the offset from a pointer (convenience for `.offset(count as isize)`). /// /// `count` is in units of T; e.g. a `count` of 3 represents a pointer @@ -2511,8 +2497,189 @@ impl *mut T { { swap(self, with) } + + /// Computes the offset that needs to be applied to the pointer in order to make it aligned to + /// `align`. + /// + /// If it is not possible to align the pointer, the implementation returns + /// `usize::max_value()`. + /// + /// The offset is expressed in number of `T` elements, and not bytes. The value returned can be + /// used with the `offset` or `offset_to` methods. + /// + /// There are no guarantees whatsover that offsetting the pointer will not overflow or go + /// beyond the allocation that the pointer points into. It is up to the caller to ensure that + /// the returned offset is correct in all terms other than alignment. + /// + /// # Panics + /// + /// The function panics if `align` is not a power-of-two. + /// + /// # Examples + /// + /// Accessing adjacent `u8` as `u16` + /// + /// ``` + /// # #![feature(align_offset)] + /// # fn foo(n: usize) { + /// # use std::mem::align_of; + /// # unsafe { + /// let x = [5u8, 6u8, 7u8, 8u8, 9u8]; + /// let ptr = &x[n] as *const u8; + /// let offset = ptr.align_offset(align_of::()); + /// if offset < x.len() - n - 1 { + /// let u16_ptr = ptr.offset(offset as isize) as *const u16; + /// assert_ne!(*u16_ptr, 500); + /// } else { + /// // while the pointer can be aligned via `offset`, it would point + /// // outside the allocation + /// } + /// # } } + /// ``` + #[unstable(feature = "align_offset", issue = "44488")] + #[cfg(not(stage0))] + pub fn align_offset(self, align: usize) -> usize where T: Sized { + if !align.is_power_of_two() { + panic!("align_offset: align is not a power-of-two"); + } + unsafe { + intrinsics::align_offset(self, align) + } + } + + /// definitely docs. + #[unstable(feature = "align_offset", issue = "44488")] + #[cfg(stage0)] + pub fn align_offset(self, align: usize) -> usize where T: Sized { + if !align.is_power_of_two() { + panic!("align_offset: align is not a power-of-two"); + } + unsafe { + intrinsics::align_offset(self as *const (), align) + } + } } +/// Align pointer `p`. +/// +/// Calculate offset (in terms of elements of `stride` stride) that has to be applied +/// to pointer `p` so that pointer `p` would get aligned to `a`. +/// +/// This is an implementation of the `align_offset` intrinsic for the case where `stride > 1`. +/// +/// Note: This implementation has been carefully tailored to not panic. It is UB for this to panic. +/// The only real change that can be made here is change of `INV_TABLE_MOD_16` and associated +/// constants. +/// +/// If we ever decide to make it possible to call the intrinsic with `a` that is not a +/// power-of-two, it will probably be more prudent to just change to a naive implementation rather +/// than trying to adapt this to accomodate that change. +/// +/// Any questions go to @nagisa. +#[lang="align_offset"] +#[cfg(not(stage0))] +unsafe fn align_offset(p: *const (), a: usize, stride: usize) -> usize { + /// Calculate multiplicative modular inverse of `x` modulo `m`. + /// + /// This implementation is tailored for align_offset and has following preconditions: + /// + /// * `m` is a power-of-two; + /// * `x < m`; (if `x ≥ m`, pass in `x % m` instead) + /// + /// Implementation of this function shall not panic. Ever. + fn mod_inv(x: usize, m: usize) -> usize { + /// Multiplicative modular inverse table modulo 2⁴ = 16. + /// + /// Note, that this table does not contain values where inverse does not exist (i.e. for + /// `0⁻¹ mod 16`, `2⁻¹ mod 16`, etc.) + static INV_TABLE_MOD_16: [usize; 8] = [1, 11, 13, 7, 9, 3, 5, 15]; + /// Modulo for which the `INV_TABLE_MOD_16` is intended. + const INV_TABLE_MOD: usize = 16; + /// INV_TABLE_MOD² + const INV_TABLE_MOD_SQUARED: usize = INV_TABLE_MOD * INV_TABLE_MOD; + + let table_inverse = INV_TABLE_MOD_16[(x & (INV_TABLE_MOD - 1)) >> 1]; + if m <= INV_TABLE_MOD { + return table_inverse & (m - 1); + } else { + // We iterate "up" using the following formula: + // + // $$ xy ≡ 1 (mod 2ⁿ) → xy (2 - xy) ≡ 1 (mod 2²ⁿ) $$ + // + // until 2²ⁿ ≥ m. Then we can reduce to our desired `m` by taking the result `mod m`. + let mut inverse = table_inverse; + let mut going_mod = INV_TABLE_MOD_SQUARED; + loop { + // y = y * (2 - xy) mod n + // + // Note, that we use wrapping operations here intentionally – the original formula + // uses e.g. subtraction `mod n`. It is entirely fine to do them `mod + // usize::max_value()` instead, because we take the result `mod n` at the end + // anyway. + inverse = inverse.wrapping_mul( + 2usize.wrapping_sub(x.wrapping_mul(inverse)) + ) & (going_mod - 1); + if going_mod > m { + return inverse & (m - 1); + } + going_mod = going_mod.wrapping_mul(going_mod); + } + } + } + + let a_minus_one = a.wrapping_sub(1); + let pmoda = p as usize & a_minus_one; + let smoda = stride & a_minus_one; + // a is power-of-two so cannot be 0. stride = 0 is handled by the intrinsic. + let gcdpow = intrinsics::cttz_nonzero(stride).min(intrinsics::cttz_nonzero(a)); + let gcd = 1usize << gcdpow; + + if pmoda == 0 { + // Already aligned. Yay! + return 0; + } + + if gcd == 1 { + // This branch solves for the variable $o$ in following linear congruence equation: + // + // ⎰ p + o ≡ 0 (mod a) # $p + o$ must be aligned to specified alignment $a$ + // ⎱ o ≡ 0 (mod s) # offset $o$ must be a multiple of stride $s$ + // + // where + // + // * a, s are co-prime + // + // This gives us the formula below: + // + // o = (a - (p mod a)) * (s⁻¹ mod a) * s + // + // The first term is “the relative alignment of p to a”, the second term is “how does + // incrementing p by one s change the relative alignment of p”, the third term is + // translating change in units of s to a byte count. + // + // Furthermore, the result produced by this solution is not “minimal”, so it is necessary + // to take the result $o mod lcm(s, a)$. Since $s$ and $a$ are co-prime (i.e. $gcd(s, a) = + // 1$) and $lcm(s, a) = s * a / gcd(s, a)$, we can replace $lcm(s, a)$ with just a $s * a$. + // + // (Author note: we decided later on to express the offset in "elements" rather than bytes, + // which drops the multiplication by `s` on both sides of the modulo.) + return intrinsics::unchecked_rem(a.wrapping_sub(pmoda).wrapping_mul(mod_inv(smoda, a)), a); + } + + if p as usize & (gcd - 1) == 0 { + // This can be aligned, but `a` and `stride` are not co-prime, so a somewhat adapted + // formula is used. + let j = a.wrapping_sub(pmoda) >> gcdpow; + let k = smoda >> gcdpow; + return intrinsics::unchecked_rem(j.wrapping_mul(mod_inv(k, a)), a >> gcdpow); + } + + // Cannot be aligned at all. + return usize::max_value(); +} + + + // Equality for pointers #[stable(feature = "rust1", since = "1.0.0")] impl PartialEq for *const T { diff --git a/src/libcore/slice/mod.rs b/src/libcore/slice/mod.rs index 82891b691dc57..6b3c1fcddc279 100644 --- a/src/libcore/slice/mod.rs +++ b/src/libcore/slice/mod.rs @@ -1696,6 +1696,28 @@ impl [T] { self.as_mut_ptr(), other.as_mut_ptr(), self.len()); } } + + // #[unstable(feature = "slice_align_to", issue = "44488")] + // pub fn align_to(&self) -> (&[T], &[U], &[T]) { + // // First, find at what point do we split between the first and 2nd slice. + // let x = self.as_ptr(); + // let offset = x.align_offset(::mem::align_of::()); + // if offset > x * ::mem::size_of::() { + // return (self, [], []); + // } + + // } + + // #[unstable(feature = "slice_align_to", issue = "44488")] + // pub fn align_to_mut(&mut self) -> (&mut [T], &mut [U], &mut [T]) { + // } +}} + +#[lang = "slice"] +#[cfg(not(test))] +#[cfg(not(stage0))] +impl [T] { + slice_core_methods!(); } #[lang = "slice_u8"] diff --git a/src/librustc/middle/lang_items.rs b/src/librustc/middle/lang_items.rs index 95e92e21b09dc..d70f994e87ba7 100644 --- a/src/librustc/middle/lang_items.rs +++ b/src/librustc/middle/lang_items.rs @@ -348,6 +348,9 @@ language_item_table! { I128ShroFnLangItem, "i128_shro", i128_shro_fn; U128ShroFnLangItem, "u128_shro", u128_shro_fn; + // Align offset for stride != 1, must not panic. + AlignOffsetLangItem, "align_offset", align_offset_fn; + TerminationTraitLangItem, "termination", termination; } diff --git a/src/librustc_codegen_llvm/intrinsic.rs b/src/librustc_codegen_llvm/intrinsic.rs index ba04cc7fad5e9..93351651db94a 100644 --- a/src/librustc_codegen_llvm/intrinsic.rs +++ b/src/librustc_codegen_llvm/intrinsic.rs @@ -25,6 +25,7 @@ use type_of::LayoutLlvmExt; use rustc::ty::{self, Ty}; use rustc::ty::layout::{HasDataLayout, LayoutOf}; use rustc::hir; +use rustc::middle::lang_items::AlignOffsetLangItem; use syntax::ast; use syntax::symbol::Symbol; use builder::Builder; @@ -390,16 +391,29 @@ pub fn codegen_intrinsic_call<'a, 'tcx>(bx: &Builder<'a, 'tcx>, } "align_offset" => { - // `ptr as usize` - let ptr_val = bx.ptrtoint(args[0].immediate(), bx.cx.isize_ty); - // `ptr_val % align` - let align = args[1].immediate(); - let offset = bx.urem(ptr_val, align); + let (ptr, align) = (args[0].immediate(), args[1].immediate()); + let stride_of_t = bx.cx.layout_of(substs.type_at(0)).size_and_align().0.bytes(); + let stride = C_usize(bx.cx, stride_of_t); let zero = C_null(bx.cx.isize_ty); - // `offset == 0` - let is_zero = bx.icmp(llvm::IntPredicate::IntEQ, offset, zero); - // `if offset == 0 { 0 } else { align - offset }` - bx.select(is_zero, zero, bx.sub(align, offset)) + let max = C_int(cx.isize_ty, -1); // -1isize (wherein I cheat horribly to make !0usize) + + if stride_of_t <= 1 { + // offset = ptr as usize % align => offset = ptr as usize & (align - 1) + let modmask = bx.sub(align, C_usize(bx.cx, 1)); + let offset = bx.and(bx.ptrtoint(ptr, bx.cx.isize_ty), modmask); + let is_zero = bx.icmp(llvm::IntPredicate::IntEQ, offset, zero); + // if offset == 0 { 0 } else { if stride_of_t == 1 { align - offset } else { !0 } } + bx.select(is_zero, zero, if stride_of_t == 1 { + bx.sub(align, offset) + } else { + max + }) + } else { + let did = ::common::langcall(bx.tcx(), Some(span), "", AlignOffsetLangItem); + let instance = ty::Instance::mono(bx.tcx(), did); + let llfn = ::callee::get_fn(bx.cx, instance); + bx.call(llfn, &[ptr, align, stride], None) + } } name if name.starts_with("simd_") => { match generic_simd_intrinsic(bx, name, diff --git a/src/librustc_typeck/check/intrinsic.rs b/src/librustc_typeck/check/intrinsic.rs index feb26e7616257..db6062db1fb1c 100644 --- a/src/librustc_typeck/check/intrinsic.rs +++ b/src/librustc_typeck/check/intrinsic.rs @@ -315,8 +315,7 @@ pub fn check_intrinsic_type<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, } "align_offset" => { - let ptr_ty = tcx.mk_imm_ptr(tcx.mk_nil()); - (0, vec![ptr_ty, tcx.types.usize], tcx.types.usize) + (1, vec![tcx.mk_imm_ptr(param(0)), tcx.types.usize], tcx.types.usize) }, "nontemporal_store" => { diff --git a/src/test/run-pass/align-offset-sign.rs b/src/test/run-pass/align-offset-sign.rs index aaa0419d061cb..f7d427cb7b26c 100644 --- a/src/test/run-pass/align-offset-sign.rs +++ b/src/test/run-pass/align-offset-sign.rs @@ -10,7 +10,85 @@ #![feature(align_offset)] +#[derive(Clone, Copy)] +#[repr(packed)] +struct A3(u16, u8); +struct A4(u32); +#[repr(packed)] +struct A5(u32, u8); +#[repr(packed)] +struct A6(u32, u16); +#[repr(packed)] +struct A7(u32, u16, u8); +#[repr(packed)] +struct A8(u32, u32); +#[repr(packed)] +struct A9(u32, u32, u8); +#[repr(packed)] +struct A10(u32, u32, u16); + +unsafe fn test_weird_stride(ptr: *const T, align: usize) -> bool { + let numptr = ptr as usize; + let mut expected = usize::max_value(); + // Naive but definitely correct way to find the *first* aligned element of stride::. + for el in (0..align) { + if (numptr + el * ::std::mem::size_of::()) % align == 0 { + expected = el; + break; + } + } + let got = ptr.align_offset(align); + if got != expected { + eprintln!("aligning {:p} (with stride of {}) to {}, expected {}, got {}", ptr, ::std::mem::size_of::(), align, expected, got); + return true; + } + return false; +} + fn main() { - let x = 1 as *const u8; - assert_eq!(x.align_offset(8), 7); + unsafe { + // For pointers of stride = 0, the pointer is already aligned or it cannot be aligned at + // all, because no amount of elements will align the pointer. + let mut p = 1; + while p < 1024 { + assert_eq!((p as *const ()).align_offset(p), 0); + if (p != 1) { + assert_eq!(((p + 1) as *const ()).align_offset(p), !0); + } + p = (p + 1).next_power_of_two(); + } + + // For pointers of stride = 1, the pointer can always be aligned. The offset is equal to + // number of bytes. + let mut align = 1; + while align < 1024 { + for ptr in 1..2*align { + let expected = ptr % align; + let offset = if expected == 0 { 0 } else { align - expected }; + assert_eq!((ptr as *const u8).align_offset(align), offset, + "ptr = {}, align = {}, size = 1", ptr, align); + align = (align + 1).next_power_of_two(); + } + } + + + // For pointers of stride != 1, we verify the algorithm against the naivest possible + // implementation + let mut align = 1; + let mut x = false; + while align < 1024 { + for ptr in 1usize..4*align { + x |= test_weird_stride::(ptr as *const A3, align); + x |= test_weird_stride::(ptr as *const A4, align); + x |= test_weird_stride::(ptr as *const A5, align); + x |= test_weird_stride::(ptr as *const A6, align); + x |= test_weird_stride::(ptr as *const A7, align); + x |= test_weird_stride::(ptr as *const A8, align); + x |= test_weird_stride::(ptr as *const A9, align); + x |= test_weird_stride::(ptr as *const A10, align); + } + align = (align + 1).next_power_of_two(); + } + assert!(!x); + } } From 680031b0164a94aaeee17a1d8c3027e6e8865a4c Mon Sep 17 00:00:00 2001 From: Simonas Kazlauskas Date: Sun, 29 Apr 2018 17:09:56 +0300 Subject: [PATCH 2/4] Implement [T]::align_to --- src/libcore/slice/mod.rs | 182 ++++++++++++++++++++++--- src/libcore/tests/lib.rs | 2 + src/libcore/tests/ptr.rs | 89 ++++++++++++ src/libcore/tests/slice.rs | 34 +++++ src/test/run-pass/align-offset-sign.rs | 94 ------------- 5 files changed, 287 insertions(+), 114 deletions(-) delete mode 100644 src/test/run-pass/align-offset-sign.rs diff --git a/src/libcore/slice/mod.rs b/src/libcore/slice/mod.rs index 6b3c1fcddc279..ffa4a66346c99 100644 --- a/src/libcore/slice/mod.rs +++ b/src/libcore/slice/mod.rs @@ -1697,27 +1697,169 @@ impl [T] { } } - // #[unstable(feature = "slice_align_to", issue = "44488")] - // pub fn align_to(&self) -> (&[T], &[U], &[T]) { - // // First, find at what point do we split between the first and 2nd slice. - // let x = self.as_ptr(); - // let offset = x.align_offset(::mem::align_of::()); - // if offset > x * ::mem::size_of::() { - // return (self, [], []); - // } - - // } - - // #[unstable(feature = "slice_align_to", issue = "44488")] - // pub fn align_to_mut(&mut self) -> (&mut [T], &mut [U], &mut [T]) { - // } -}} + /// Function to calculate lenghts of the middle and trailing slice for `align_to{,_mut}`. + fn align_to_offsets(&self) -> (usize, usize) { + // What we gonna do about `rest` is figure out what multiple of `U`s we can put in a + // lowest number of `T`s. And how many `T`s we need for each such "multiple". + // + // Consider for example T=u8 U=u16. Then we can put 1 U in 2 Ts. Simple. Now, consider + // for example a case where size_of:: = 16, size_of:: = 24. We can put 2 Us in + // place of every 3 Ts in the `rest` slice. A bit more complicated. + // + // Formula to calculate this is: + // + // Us = lcm(size_of::, size_of::) / size_of:: + // Ts = lcm(size_of::, size_of::) / size_of:: + // + // Expanded and simplified: + // + // Us = size_of:: / gcd(size_of::, size_of::) + // Ts = size_of:: / gcd(size_of::, size_of::) + // + // Luckily since all this is constant-evaluated... performance here matters not! + #[inline] + fn gcd(a: usize, b: usize) -> usize { + // iterative stein’s algorithm + // We should still make this `const fn` (and revert to recursive algorithm if we do) + // because relying on llvm to consteval all this is… well, it makes me + let (ctz_a, mut ctz_b) = unsafe { + if a == 0 { return b; } + if b == 0 { return a; } + (::intrinsics::cttz_nonzero(a), ::intrinsics::cttz_nonzero(b)) + }; + let k = ctz_a.min(ctz_b); + let mut a = a >> ctz_a; + let mut b = b; + loop { + // remove all factors of 2 from b + b >>= ctz_b; + if a > b { + ::mem::swap(&mut a, &mut b); + } + b = b - a; + unsafe { + if b == 0 { + break; + } + ctz_b = ::intrinsics::cttz_nonzero(b); + } + } + return a << k; + } + let gcd: usize = gcd(::mem::size_of::(), ::mem::size_of::()); + let ts: usize = ::mem::size_of::() / gcd; + let us: usize = ::mem::size_of::() / gcd; -#[lang = "slice"] -#[cfg(not(test))] -#[cfg(not(stage0))] -impl [T] { - slice_core_methods!(); + // Armed with this knowledge, we can find how many `U`s we can fit! + let us_len = self.len() / ts * us; + // And how many `T`s will be in the trailing slice! + let ts_len = self.len() % ts; + return (us_len, ts_len); + } + + /// Transmute the slice to a slice of another type, ensuring aligment of the types is + /// maintained. + /// + /// This method splits the slice into three distinct slices: prefix, correctly aligned middle + /// slice of a new type, and the suffix slice. The middle slice will have the greatest length + /// possible for a given type and input slice. + /// + /// This method has no purpose when either input element `T` or output element `U` are + /// zero-sized and will return the original slice without splitting anything. + /// + /// # Unsafety + /// + /// This method is essentially a `transmute` with respect to the elements in the returned + /// middle slice, so all the usual caveats pertaining to `transmute::` also apply here. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// # #![feature(slice_align_to)] + /// unsafe { + /// let bytes: [u8; 7] = [1, 2, 3, 4, 5, 6, 7]; + /// let (prefix, shorts, suffix) = bytes.align_to::(); + /// // less_efficient_algorithm_for_bytes(prefix); + /// // more_efficient_algorithm_for_aligned_shorts(shorts); + /// // less_efficient_algorithm_for_bytes(suffix); + /// } + /// ``` + #[unstable(feature = "slice_align_to", issue = "44488")] + #[cfg(not(stage0))] + pub unsafe fn align_to(&self) -> (&[T], &[U], &[T]) { + // Note that most of this function will be constant-evaluated, + if ::mem::size_of::() == 0 || ::mem::size_of::() == 0 { + // handle ZSTs specially, which is – don't handle them at all. + return (self, &[], &[]); + } + let ptr = self.as_ptr(); + let offset = ::intrinsics::align_offset(ptr, ::mem::align_of::()); + if offset > self.len() { + return (self, &[], &[]); + } else { + let (left, rest) = self.split_at(offset); + let (us_len, ts_len) = rest.align_to_offsets::(); + return (left, + from_raw_parts(rest.as_ptr() as *const U, us_len), + from_raw_parts(rest.as_ptr().offset((rest.len() - ts_len) as isize), ts_len)) + } + } + + /// Transmute the slice to a slice of another type, ensuring aligment of the types is + /// maintained. + /// + /// This method splits the slice into three distinct slices: prefix, correctly aligned middle + /// slice of a new type, and the suffix slice. The middle slice will have the greatest length + /// possible for a given type and input slice. + /// + /// This method has no purpose when either input element `T` or output element `U` are + /// zero-sized and will return the original slice without splitting anything. + /// + /// # Unsafety + /// + /// This method is essentially a `transmute` with respect to the elements in the returned + /// middle slice, so all the usual caveats pertaining to `transmute::` also apply here. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// # #![feature(slice_align_to)] + /// unsafe { + /// let mut bytes: [u8; 7] = [1, 2, 3, 4, 5, 6, 7]; + /// let (prefix, shorts, suffix) = bytes.align_to_mut::(); + /// // less_efficient_algorithm_for_bytes(prefix); + /// // more_efficient_algorithm_for_aligned_shorts(shorts); + /// // less_efficient_algorithm_for_bytes(suffix); + /// } + /// ``` + #[unstable(feature = "slice_align_to", issue = "44488")] + #[cfg(not(stage0))] + pub unsafe fn align_to_mut(&mut self) -> (&mut [T], &mut [U], &mut [T]) { + // Note that most of this function will be constant-evaluated, + if ::mem::size_of::() == 0 || ::mem::size_of::() == 0 { + // handle ZSTs specially, which is – don't handle them at all. + return (self, &mut [], &mut []); + } + + // First, find at what point do we split between the first and 2nd slice. Easy with + // ptr.align_offset. + let ptr = self.as_ptr(); + let offset = ::intrinsics::align_offset(ptr, ::mem::align_of::()); + if offset > self.len() { + return (self, &mut [], &mut []); + } else { + let (left, rest) = self.split_at_mut(offset); + let (us_len, ts_len) = rest.align_to_offsets::(); + let mut_ptr = rest.as_mut_ptr(); + return (left, + from_raw_parts_mut(mut_ptr as *mut U, us_len), + from_raw_parts_mut(mut_ptr.offset((rest.len() - ts_len) as isize), ts_len)) + } + } } #[lang = "slice_u8"] diff --git a/src/libcore/tests/lib.rs b/src/libcore/tests/lib.rs index 8c481338945ff..dbd26b2c71836 100644 --- a/src/libcore/tests/lib.rs +++ b/src/libcore/tests/lib.rs @@ -41,6 +41,8 @@ #![feature(try_from)] #![feature(try_trait)] #![feature(exact_chunks)] +#![feature(slice_align_to)] +#![feature(align_offset)] #![feature(reverse_bits)] #![feature(inclusive_range_methods)] #![feature(iterator_find_map)] diff --git a/src/libcore/tests/ptr.rs b/src/libcore/tests/ptr.rs index 00f87336f3c80..9384cb327981d 100644 --- a/src/libcore/tests/ptr.rs +++ b/src/libcore/tests/ptr.rs @@ -296,3 +296,92 @@ fn write_unaligned_drop() { } DROPS.with(|d| assert_eq!(*d.borrow(), [0])); } + +#[test] +fn align_offset_zst() { + // For pointers of stride = 0, the pointer is already aligned or it cannot be aligned at + // all, because no amount of elements will align the pointer. + let mut p = 1; + while p < 1024 { + assert_eq!((p as *const ()).align_offset(p), 0); + if p != 1 { + assert_eq!(((p + 1) as *const ()).align_offset(p), !0); + } + p = (p + 1).next_power_of_two(); + } +} + +#[test] +fn align_offset_stride1() { + // For pointers of stride = 1, the pointer can always be aligned. The offset is equal to + // number of bytes. + let mut align = 1; + while align < 1024 { + for ptr in 1..2*align { + let expected = ptr % align; + let offset = if expected == 0 { 0 } else { align - expected }; + assert_eq!((ptr as *const u8).align_offset(align), offset, + "ptr = {}, align = {}, size = 1", ptr, align); + align = (align + 1).next_power_of_two(); + } + } +} + +#[test] +fn align_offset_weird_strides() { + #[repr(packed)] + struct A3(u16, u8); + struct A4(u32); + #[repr(packed)] + struct A5(u32, u8); + #[repr(packed)] + struct A6(u32, u16); + #[repr(packed)] + struct A7(u32, u16, u8); + #[repr(packed)] + struct A8(u32, u32); + #[repr(packed)] + struct A9(u32, u32, u8); + #[repr(packed)] + struct A10(u32, u32, u16); + + unsafe fn test_weird_stride(ptr: *const T, align: usize) -> bool { + let numptr = ptr as usize; + let mut expected = usize::max_value(); + // Naive but definitely correct way to find the *first* aligned element of stride::. + for el in 0..align { + if (numptr + el * ::std::mem::size_of::()) % align == 0 { + expected = el; + break; + } + } + let got = ptr.align_offset(align); + if got != expected { + eprintln!("aligning {:p} (with stride of {}) to {}, expected {}, got {}", ptr, + ::std::mem::size_of::(), align, expected, got); + return true; + } + return false; + } + + // For pointers of stride != 1, we verify the algorithm against the naivest possible + // implementation + let mut align = 1; + let mut x = false; + while align < 1024 { + for ptr in 1usize..4*align { + unsafe { + x |= test_weird_stride::(ptr as *const A3, align); + x |= test_weird_stride::(ptr as *const A4, align); + x |= test_weird_stride::(ptr as *const A5, align); + x |= test_weird_stride::(ptr as *const A6, align); + x |= test_weird_stride::(ptr as *const A7, align); + x |= test_weird_stride::(ptr as *const A8, align); + x |= test_weird_stride::(ptr as *const A9, align); + x |= test_weird_stride::(ptr as *const A10, align); + } + } + align = (align + 1).next_power_of_two(); + } + assert!(!x); +} diff --git a/src/libcore/tests/slice.rs b/src/libcore/tests/slice.rs index 19b5c86c47406..8acb531b989a0 100644 --- a/src/libcore/tests/slice.rs +++ b/src/libcore/tests/slice.rs @@ -812,3 +812,37 @@ pub mod memchr { } } } + +#[test] +fn test_align_to_simple() { + let bytes = [1u8, 2, 3, 4, 5, 6, 7]; + let (prefix, aligned, suffix) = unsafe { bytes.align_to::() }; + assert_eq!(aligned.len(), 3); + assert!(prefix == [1] || suffix == [7]); + let expect1 = [1 << 8 | 2, 3 << 8 | 4, 5 << 8 | 6]; + let expect2 = [1 | 2 << 8, 3 | 4 << 8, 5 | 6 << 8]; + let expect3 = [2 | 3 << 8, 4 | 5 << 8, 6 | 7 << 8]; + let expect4 = [2 | 3 << 8, 4 | 5 << 8, 6 | 7 << 8]; + assert!(aligned == expect1 || aligned == expect2 || aligned == expect3 || aligned == expect4, + "aligned={:?} expected={:?} || {:?} || {:?} || {:?}", + aligned, expect1, expect2, expect3, expect4); +} + +#[test] +fn test_align_to_zst() { + let bytes = [1, 2, 3, 4, 5, 6, 7]; + let (prefix, aligned, suffix) = unsafe { bytes.align_to::<()>() }; + assert_eq!(aligned.len(), 0); + assert!(prefix == [1, 2, 3, 4, 5, 6, 7] || suffix == [1, 2, 3, 4, 5, 6, 7]); +} + +#[test] +fn test_align_to_non_trivial() { + #[repr(align(8))] struct U64(u64, u64); + #[repr(align(8))] struct U64U64U32(u64, u64, u32); + let data = [U64(1, 2), U64(3, 4), U64(5, 6), U64(7, 8), U64(9, 10), U64(11, 12), U64(13, 14), + U64(15, 16)]; + let (prefix, aligned, suffix) = unsafe { data.align_to::() }; + assert_eq!(aligned.len(), 4); + assert_eq!(prefix.len() + suffix.len(), 2); +} diff --git a/src/test/run-pass/align-offset-sign.rs b/src/test/run-pass/align-offset-sign.rs deleted file mode 100644 index f7d427cb7b26c..0000000000000 --- a/src/test/run-pass/align-offset-sign.rs +++ /dev/null @@ -1,94 +0,0 @@ -// Copyright 2017 The Rust Project Developers. See the COPYRIGHT -// file at the top-level directory of this distribution and at -// http://rust-lang.org/COPYRIGHT. -// -// Licensed under the Apache License, Version 2.0 or the MIT license -// , at your -// option. This file may not be copied, modified, or distributed -// except according to those terms. - -#![feature(align_offset)] - -#[derive(Clone, Copy)] -#[repr(packed)] -struct A3(u16, u8); -struct A4(u32); -#[repr(packed)] -struct A5(u32, u8); -#[repr(packed)] -struct A6(u32, u16); -#[repr(packed)] -struct A7(u32, u16, u8); -#[repr(packed)] -struct A8(u32, u32); -#[repr(packed)] -struct A9(u32, u32, u8); -#[repr(packed)] -struct A10(u32, u32, u16); - -unsafe fn test_weird_stride(ptr: *const T, align: usize) -> bool { - let numptr = ptr as usize; - let mut expected = usize::max_value(); - // Naive but definitely correct way to find the *first* aligned element of stride::. - for el in (0..align) { - if (numptr + el * ::std::mem::size_of::()) % align == 0 { - expected = el; - break; - } - } - let got = ptr.align_offset(align); - if got != expected { - eprintln!("aligning {:p} (with stride of {}) to {}, expected {}, got {}", ptr, ::std::mem::size_of::(), align, expected, got); - return true; - } - return false; -} - -fn main() { - unsafe { - // For pointers of stride = 0, the pointer is already aligned or it cannot be aligned at - // all, because no amount of elements will align the pointer. - let mut p = 1; - while p < 1024 { - assert_eq!((p as *const ()).align_offset(p), 0); - if (p != 1) { - assert_eq!(((p + 1) as *const ()).align_offset(p), !0); - } - p = (p + 1).next_power_of_two(); - } - - // For pointers of stride = 1, the pointer can always be aligned. The offset is equal to - // number of bytes. - let mut align = 1; - while align < 1024 { - for ptr in 1..2*align { - let expected = ptr % align; - let offset = if expected == 0 { 0 } else { align - expected }; - assert_eq!((ptr as *const u8).align_offset(align), offset, - "ptr = {}, align = {}, size = 1", ptr, align); - align = (align + 1).next_power_of_two(); - } - } - - - // For pointers of stride != 1, we verify the algorithm against the naivest possible - // implementation - let mut align = 1; - let mut x = false; - while align < 1024 { - for ptr in 1usize..4*align { - x |= test_weird_stride::(ptr as *const A3, align); - x |= test_weird_stride::(ptr as *const A4, align); - x |= test_weird_stride::(ptr as *const A5, align); - x |= test_weird_stride::(ptr as *const A6, align); - x |= test_weird_stride::(ptr as *const A7, align); - x |= test_weird_stride::(ptr as *const A8, align); - x |= test_weird_stride::(ptr as *const A9, align); - x |= test_weird_stride::(ptr as *const A10, align); - } - align = (align + 1).next_power_of_two(); - } - assert!(!x); - } -} From 6d5bf8b23f4e59fdc67f635a10f450dfab2f076b Mon Sep 17 00:00:00 2001 From: Simonas Kazlauskas Date: Fri, 4 May 2018 00:49:22 +0300 Subject: [PATCH 3/4] Remove the intrinsic for align_offset Keep only the language item. This removes some indirection and makes codegen worse for debug builds, but simplifies code significantly, which is a good tradeoff to make, in my opinion. Besides, the codegen can be improved even further with some constant evaluation improvements that we expect to happen in the future. --- src/libcore/intrinsics.rs | 42 -------------------------- src/libcore/ptr.rs | 31 +++++++++++++------ src/libcore/slice/mod.rs | 7 +++-- src/librustc_codegen_llvm/intrinsic.rs | 26 ---------------- src/librustc_typeck/check/intrinsic.rs | 4 --- 5 files changed, 26 insertions(+), 84 deletions(-) diff --git a/src/libcore/intrinsics.rs b/src/libcore/intrinsics.rs index 510a5bb3df7ba..1420e00b47ffa 100644 --- a/src/libcore/intrinsics.rs +++ b/src/libcore/intrinsics.rs @@ -1467,48 +1467,6 @@ extern "rust-intrinsic" { /// docs my friends, its friday! pub fn align_offset(ptr: *const (), align: usize) -> usize; - /// Computes the offset that needs to be applied to the pointer in order to make it aligned to - /// `align`. - /// - /// If it is not possible to align the pointer, the implementation returns - /// `usize::max_value()`. - /// - /// The offset is expressed in number of `T` elements, and not bytes. The value returned can be - /// used with the `offset` or `offset_to` methods. - /// - /// There are no guarantees whatsover that offsetting the pointer will not overflow or go - /// beyond the allocation that the pointer points into. It is up to the caller to ensure that - /// the returned offset is correct in all terms other than alignment. - /// - /// # Unsafety - /// - /// `align` must be a power-of-two. - /// - /// # Examples - /// - /// Accessing adjacent `u8` as `u16` - /// - /// ``` - /// # #![feature(core_intrinsics)] - /// # fn foo(n: usize) { - /// # use std::intrinsics::align_offset; - /// # use std::mem::align_of; - /// # unsafe { - /// let x = [5u8, 6u8, 7u8, 8u8, 9u8]; - /// let ptr = &x[n] as *const u8; - /// let offset = align_offset(ptr, align_of::()); - /// if offset < x.len() - n - 1 { - /// let u16_ptr = ptr.offset(offset as isize) as *const u16; - /// assert_ne!(*u16_ptr, 500); - /// } else { - /// // while the pointer can be aligned via `offset`, it would point - /// // outside the allocation - /// } - /// # } } - /// ``` - #[cfg(not(stage0))] - pub fn align_offset(ptr: *const T, align: usize) -> usize; - /// Emits a `!nontemporal` store according to LLVM (see their docs). /// Probably will never become stable. pub fn nontemporal_store(ptr: *mut T, val: T); diff --git a/src/libcore/ptr.rs b/src/libcore/ptr.rs index a762a8a6f924d..e2286d23e3348 100644 --- a/src/libcore/ptr.rs +++ b/src/libcore/ptr.rs @@ -1478,7 +1478,7 @@ impl *const T { panic!("align_offset: align is not a power-of-two"); } unsafe { - intrinsics::align_offset(self, align) + align_offset(self, align) } } @@ -2543,7 +2543,7 @@ impl *mut T { panic!("align_offset: align is not a power-of-two"); } unsafe { - intrinsics::align_offset(self, align) + align_offset(self, align) } } @@ -2565,8 +2565,6 @@ impl *mut T { /// Calculate offset (in terms of elements of `stride` stride) that has to be applied /// to pointer `p` so that pointer `p` would get aligned to `a`. /// -/// This is an implementation of the `align_offset` intrinsic for the case where `stride > 1`. -/// /// Note: This implementation has been carefully tailored to not panic. It is UB for this to panic. /// The only real change that can be made here is change of `INV_TABLE_MOD_16` and associated /// constants. @@ -2578,7 +2576,7 @@ impl *mut T { /// Any questions go to @nagisa. #[lang="align_offset"] #[cfg(not(stage0))] -unsafe fn align_offset(p: *const (), a: usize, stride: usize) -> usize { +pub(crate) unsafe fn align_offset(p: *const T, a: usize) -> usize { /// Calculate multiplicative modular inverse of `x` modulo `m`. /// /// This implementation is tailored for align_offset and has following preconditions: @@ -2587,12 +2585,13 @@ unsafe fn align_offset(p: *const (), a: usize, stride: usize) -> usize { /// * `x < m`; (if `x ≥ m`, pass in `x % m` instead) /// /// Implementation of this function shall not panic. Ever. + #[inline] fn mod_inv(x: usize, m: usize) -> usize { /// Multiplicative modular inverse table modulo 2⁴ = 16. /// /// Note, that this table does not contain values where inverse does not exist (i.e. for /// `0⁻¹ mod 16`, `2⁻¹ mod 16`, etc.) - static INV_TABLE_MOD_16: [usize; 8] = [1, 11, 13, 7, 9, 3, 5, 15]; + const INV_TABLE_MOD_16: [usize; 8] = [1, 11, 13, 7, 9, 3, 5, 15]; /// Modulo for which the `INV_TABLE_MOD_16` is intended. const INV_TABLE_MOD: usize = 16; /// INV_TABLE_MOD² @@ -2627,18 +2626,30 @@ unsafe fn align_offset(p: *const (), a: usize, stride: usize) -> usize { } } + let stride = ::mem::size_of::(); let a_minus_one = a.wrapping_sub(1); let pmoda = p as usize & a_minus_one; - let smoda = stride & a_minus_one; - // a is power-of-two so cannot be 0. stride = 0 is handled by the intrinsic. - let gcdpow = intrinsics::cttz_nonzero(stride).min(intrinsics::cttz_nonzero(a)); - let gcd = 1usize << gcdpow; if pmoda == 0 { // Already aligned. Yay! return 0; } + if stride <= 1 { + return if stride == 0 { + // If the pointer is not aligned, and the element is zero-sized, then no amount of + // elements will ever align the pointer. + !0 + } else { + a.wrapping_sub(pmoda) + }; + } + + let smoda = stride & a_minus_one; + // a is power-of-two so cannot be 0. stride = 0 is handled above. + let gcdpow = intrinsics::cttz_nonzero(stride).min(intrinsics::cttz_nonzero(a)); + let gcd = 1usize << gcdpow; + if gcd == 1 { // This branch solves for the variable $o$ in following linear congruence equation: // diff --git a/src/libcore/slice/mod.rs b/src/libcore/slice/mod.rs index ffa4a66346c99..fdc9aa473e8b8 100644 --- a/src/libcore/slice/mod.rs +++ b/src/libcore/slice/mod.rs @@ -1794,8 +1794,11 @@ impl [T] { // handle ZSTs specially, which is – don't handle them at all. return (self, &[], &[]); } + + // First, find at what point do we split between the first and 2nd slice. Easy with + // ptr.align_offset. let ptr = self.as_ptr(); - let offset = ::intrinsics::align_offset(ptr, ::mem::align_of::()); + let offset = ::ptr::align_offset(ptr, ::mem::align_of::()); if offset > self.len() { return (self, &[], &[]); } else { @@ -1848,7 +1851,7 @@ impl [T] { // First, find at what point do we split between the first and 2nd slice. Easy with // ptr.align_offset. let ptr = self.as_ptr(); - let offset = ::intrinsics::align_offset(ptr, ::mem::align_of::()); + let offset = ::ptr::align_offset(ptr, ::mem::align_of::()); if offset > self.len() { return (self, &mut [], &mut []); } else { diff --git a/src/librustc_codegen_llvm/intrinsic.rs b/src/librustc_codegen_llvm/intrinsic.rs index 93351651db94a..cffe7f79e970c 100644 --- a/src/librustc_codegen_llvm/intrinsic.rs +++ b/src/librustc_codegen_llvm/intrinsic.rs @@ -25,7 +25,6 @@ use type_of::LayoutLlvmExt; use rustc::ty::{self, Ty}; use rustc::ty::layout::{HasDataLayout, LayoutOf}; use rustc::hir; -use rustc::middle::lang_items::AlignOffsetLangItem; use syntax::ast; use syntax::symbol::Symbol; use builder::Builder; @@ -390,31 +389,6 @@ pub fn codegen_intrinsic_call<'a, 'tcx>(bx: &Builder<'a, 'tcx>, args[0].deref(bx.cx).codegen_get_discr(bx, ret_ty) } - "align_offset" => { - let (ptr, align) = (args[0].immediate(), args[1].immediate()); - let stride_of_t = bx.cx.layout_of(substs.type_at(0)).size_and_align().0.bytes(); - let stride = C_usize(bx.cx, stride_of_t); - let zero = C_null(bx.cx.isize_ty); - let max = C_int(cx.isize_ty, -1); // -1isize (wherein I cheat horribly to make !0usize) - - if stride_of_t <= 1 { - // offset = ptr as usize % align => offset = ptr as usize & (align - 1) - let modmask = bx.sub(align, C_usize(bx.cx, 1)); - let offset = bx.and(bx.ptrtoint(ptr, bx.cx.isize_ty), modmask); - let is_zero = bx.icmp(llvm::IntPredicate::IntEQ, offset, zero); - // if offset == 0 { 0 } else { if stride_of_t == 1 { align - offset } else { !0 } } - bx.select(is_zero, zero, if stride_of_t == 1 { - bx.sub(align, offset) - } else { - max - }) - } else { - let did = ::common::langcall(bx.tcx(), Some(span), "", AlignOffsetLangItem); - let instance = ty::Instance::mono(bx.tcx(), did); - let llfn = ::callee::get_fn(bx.cx, instance); - bx.call(llfn, &[ptr, align, stride], None) - } - } name if name.starts_with("simd_") => { match generic_simd_intrinsic(bx, name, callee_ty, diff --git a/src/librustc_typeck/check/intrinsic.rs b/src/librustc_typeck/check/intrinsic.rs index db6062db1fb1c..af1f1044edf2f 100644 --- a/src/librustc_typeck/check/intrinsic.rs +++ b/src/librustc_typeck/check/intrinsic.rs @@ -314,10 +314,6 @@ pub fn check_intrinsic_type<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, (0, vec![tcx.mk_fn_ptr(fn_ty), mut_u8, mut_u8], tcx.types.i32) } - "align_offset" => { - (1, vec![tcx.mk_imm_ptr(param(0)), tcx.types.usize], tcx.types.usize) - }, - "nontemporal_store" => { (1, vec![ tcx.mk_mut_ptr(param(0)), param(0) ], tcx.mk_nil()) } From 59bb0fe66ea6d80c5e1466b9609887e4cf7cde47 Mon Sep 17 00:00:00 2001 From: Simonas Kazlauskas Date: Thu, 17 May 2018 18:02:47 +0300 Subject: [PATCH 4/4] Fix align_offset_stride1 & align_to_simple tests --- src/libcore/tests/ptr.rs | 2 +- src/libcore/tests/slice.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libcore/tests/ptr.rs b/src/libcore/tests/ptr.rs index 9384cb327981d..31bc1d677686a 100644 --- a/src/libcore/tests/ptr.rs +++ b/src/libcore/tests/ptr.rs @@ -322,8 +322,8 @@ fn align_offset_stride1() { let offset = if expected == 0 { 0 } else { align - expected }; assert_eq!((ptr as *const u8).align_offset(align), offset, "ptr = {}, align = {}, size = 1", ptr, align); - align = (align + 1).next_power_of_two(); } + align = (align + 1).next_power_of_two(); } } diff --git a/src/libcore/tests/slice.rs b/src/libcore/tests/slice.rs index 8acb531b989a0..fcd79222e1699 100644 --- a/src/libcore/tests/slice.rs +++ b/src/libcore/tests/slice.rs @@ -821,7 +821,7 @@ fn test_align_to_simple() { assert!(prefix == [1] || suffix == [7]); let expect1 = [1 << 8 | 2, 3 << 8 | 4, 5 << 8 | 6]; let expect2 = [1 | 2 << 8, 3 | 4 << 8, 5 | 6 << 8]; - let expect3 = [2 | 3 << 8, 4 | 5 << 8, 6 | 7 << 8]; + let expect3 = [2 << 8 | 3, 4 << 8 | 5, 6 << 8 | 7]; let expect4 = [2 | 3 << 8, 4 | 5 << 8, 6 | 7 << 8]; assert!(aligned == expect1 || aligned == expect2 || aligned == expect3 || aligned == expect4, "aligned={:?} expected={:?} || {:?} || {:?} || {:?}",