diff --git a/compiler/rustc_codegen_cranelift/src/codegen_i128.rs b/compiler/rustc_codegen_cranelift/src/codegen_i128.rs index b2bc289a5b6ba..4a5ef352151f3 100644 --- a/compiler/rustc_codegen_cranelift/src/codegen_i128.rs +++ b/compiler/rustc_codegen_cranelift/src/codegen_i128.rs @@ -68,7 +68,7 @@ pub(crate) fn maybe_codegen<'tcx>( Some(CValue::by_val(ret_val, lhs.layout())) } } - BinOp::Lt | BinOp::Le | BinOp::Eq | BinOp::Ge | BinOp::Gt | BinOp::Ne => None, + BinOp::Lt | BinOp::Le | BinOp::Eq | BinOp::Ge | BinOp::Gt | BinOp::Ne | BinOp::Cmp => None, BinOp::Shl | BinOp::ShlUnchecked | BinOp::Shr | BinOp::ShrUnchecked => None, } } @@ -134,6 +134,7 @@ pub(crate) fn maybe_codegen_checked<'tcx>( BinOp::AddUnchecked | BinOp::SubUnchecked | BinOp::MulUnchecked => unreachable!(), BinOp::Offset => unreachable!("offset should only be used on pointers, not 128bit ints"), BinOp::Div | BinOp::Rem => unreachable!(), + BinOp::Cmp => unreachable!(), BinOp::Lt | BinOp::Le | BinOp::Eq | BinOp::Ge | BinOp::Gt | BinOp::Ne => unreachable!(), BinOp::Shl | BinOp::ShlUnchecked | BinOp::Shr | BinOp::ShrUnchecked => unreachable!(), } diff --git a/compiler/rustc_codegen_cranelift/src/num.rs b/compiler/rustc_codegen_cranelift/src/num.rs index 8992f40fb903a..796182418ad6c 100644 --- a/compiler/rustc_codegen_cranelift/src/num.rs +++ b/compiler/rustc_codegen_cranelift/src/num.rs @@ -40,6 +40,22 @@ pub(crate) fn bin_op_to_intcc(bin_op: BinOp, signed: bool) -> Option { }) } +fn codegen_three_way_compare<'tcx>( + fx: &mut FunctionCx<'_, '_, 'tcx>, + signed: bool, + lhs: Value, + rhs: Value, +) -> CValue<'tcx> { + // This emits `(lhs > rhs) - (lhs < rhs)`, which is cranelift's preferred form per + // + let gt_cc = crate::num::bin_op_to_intcc(BinOp::Gt, signed).unwrap(); + let lt_cc = crate::num::bin_op_to_intcc(BinOp::Lt, signed).unwrap(); + let gt = fx.bcx.ins().icmp(gt_cc, lhs, rhs); + let lt = fx.bcx.ins().icmp(lt_cc, lhs, rhs); + let val = fx.bcx.ins().isub(gt, lt); + CValue::by_val(val, fx.layout_of(fx.tcx.ty_ordering_enum(Some(fx.mir.span)))) +} + fn codegen_compare_bin_op<'tcx>( fx: &mut FunctionCx<'_, '_, 'tcx>, bin_op: BinOp, @@ -47,6 +63,10 @@ fn codegen_compare_bin_op<'tcx>( lhs: Value, rhs: Value, ) -> CValue<'tcx> { + if bin_op == BinOp::Cmp { + return codegen_three_way_compare(fx, signed, lhs, rhs); + } + let intcc = crate::num::bin_op_to_intcc(bin_op, signed).unwrap(); let val = fx.bcx.ins().icmp(intcc, lhs, rhs); CValue::by_val(val, fx.layout_of(fx.tcx.types.bool)) @@ -59,7 +79,7 @@ pub(crate) fn codegen_binop<'tcx>( in_rhs: CValue<'tcx>, ) -> CValue<'tcx> { match bin_op { - BinOp::Eq | BinOp::Lt | BinOp::Le | BinOp::Ne | BinOp::Ge | BinOp::Gt => { + BinOp::Eq | BinOp::Lt | BinOp::Le | BinOp::Ne | BinOp::Ge | BinOp::Gt | BinOp::Cmp => { match in_lhs.layout().ty.kind() { ty::Bool | ty::Uint(_) | ty::Int(_) | ty::Char => { let signed = type_sign(in_lhs.layout().ty); @@ -160,7 +180,7 @@ pub(crate) fn codegen_int_binop<'tcx>( } BinOp::Offset => unreachable!("Offset is not an integer operation"), // Compare binops handles by `codegen_binop`. - BinOp::Eq | BinOp::Ne | BinOp::Lt | BinOp::Le | BinOp::Gt | BinOp::Ge => { + BinOp::Eq | BinOp::Ne | BinOp::Lt | BinOp::Le | BinOp::Gt | BinOp::Ge | BinOp::Cmp => { unreachable!("{:?}({:?}, {:?})", bin_op, in_lhs.layout().ty, in_rhs.layout().ty); } }; diff --git a/compiler/rustc_codegen_gcc/src/common.rs b/compiler/rustc_codegen_gcc/src/common.rs index d243d7088ada9..78d943192db07 100644 --- a/compiler/rustc_codegen_gcc/src/common.rs +++ b/compiler/rustc_codegen_gcc/src/common.rs @@ -94,6 +94,10 @@ impl<'gcc, 'tcx> ConstMethods<'tcx> for CodegenCx<'gcc, 'tcx> { self.const_int(self.type_i32(), i as i64) } + fn const_i8(&self, i: i8) -> RValue<'gcc> { + self.const_int(self.type_i8(), i as i64) + } + fn const_u32(&self, i: u32) -> RValue<'gcc> { self.const_uint(self.type_u32(), i as u64) } diff --git a/compiler/rustc_codegen_llvm/src/common.rs b/compiler/rustc_codegen_llvm/src/common.rs index 25cbd90460f27..568fcc3f3cf49 100644 --- a/compiler/rustc_codegen_llvm/src/common.rs +++ b/compiler/rustc_codegen_llvm/src/common.rs @@ -160,6 +160,10 @@ impl<'ll, 'tcx> ConstMethods<'tcx> for CodegenCx<'ll, 'tcx> { self.const_int(self.type_i32(), i as i64) } + fn const_i8(&self, i: i8) -> &'ll Value { + self.const_int(self.type_i8(), i as i64) + } + fn const_u32(&self, i: u32) -> &'ll Value { self.const_uint(self.type_i32(), i as u64) } diff --git a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs index 0e8c4abf21264..f0933fe33d747 100644 --- a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs +++ b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs @@ -7,6 +7,7 @@ use crate::common::{self, IntPredicate}; use crate::traits::*; use crate::MemFlags; +use rustc_hir as hir; use rustc_middle::mir; use rustc_middle::mir::Operand; use rustc_middle::ty::cast::{CastTy, IntTy}; @@ -882,6 +883,35 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { bx.icmp(base::bin_op_to_icmp_predicate(op.to_hir_binop(), is_signed), lhs, rhs) } } + mir::BinOp::Cmp => { + use std::cmp::Ordering; + debug_assert!(!is_float); + let pred = |op| base::bin_op_to_icmp_predicate(op, is_signed); + if bx.cx().tcx().sess.opts.optimize == OptLevel::No { + // FIXME: This actually generates tighter assembly, and is a classic trick + // + // However, as of 2023-11 it optimizes worse in things like derived + // `PartialOrd`, so only use it in debug for now. Once LLVM can handle it + // better (see ), it'll + // be worth trying it in optimized builds as well. + let is_gt = bx.icmp(pred(hir::BinOpKind::Gt), lhs, rhs); + let gtext = bx.zext(is_gt, bx.type_i8()); + let is_lt = bx.icmp(pred(hir::BinOpKind::Lt), lhs, rhs); + let ltext = bx.zext(is_lt, bx.type_i8()); + bx.unchecked_ssub(gtext, ltext) + } else { + // These operations are those expected by `tests/codegen/integer-cmp.rs`, + // from . + let is_lt = bx.icmp(pred(hir::BinOpKind::Lt), lhs, rhs); + let is_ne = bx.icmp(pred(hir::BinOpKind::Ne), lhs, rhs); + let ge = bx.select( + is_ne, + bx.cx().const_i8(Ordering::Greater as i8), + bx.cx().const_i8(Ordering::Equal as i8), + ); + bx.select(is_lt, bx.cx().const_i8(Ordering::Less as i8), ge) + } + } } } diff --git a/compiler/rustc_codegen_ssa/src/traits/consts.rs b/compiler/rustc_codegen_ssa/src/traits/consts.rs index 4dff9c7684f18..8cb17a5b37a89 100644 --- a/compiler/rustc_codegen_ssa/src/traits/consts.rs +++ b/compiler/rustc_codegen_ssa/src/traits/consts.rs @@ -19,6 +19,7 @@ pub trait ConstMethods<'tcx>: BackendTypes { fn const_bool(&self, val: bool) -> Self::Value; fn const_i16(&self, i: i16) -> Self::Value; fn const_i32(&self, i: i32) -> Self::Value; + fn const_i8(&self, i: i8) -> Self::Value; fn const_u32(&self, i: u32) -> Self::Value; fn const_u64(&self, i: u64) -> Self::Value; fn const_u128(&self, i: u128) -> Self::Value; diff --git a/compiler/rustc_const_eval/src/interpret/operand.rs b/compiler/rustc_const_eval/src/interpret/operand.rs index dbc6a317640c1..831787a92c869 100644 --- a/compiler/rustc_const_eval/src/interpret/operand.rs +++ b/compiler/rustc_const_eval/src/interpret/operand.rs @@ -235,6 +235,13 @@ impl<'tcx, Prov: Provenance> ImmTy<'tcx, Prov> { Self::from_scalar(Scalar::from_bool(b), layout) } + #[inline] + pub fn from_ordering(c: std::cmp::Ordering, tcx: TyCtxt<'tcx>) -> Self { + let ty = tcx.ty_ordering_enum(None); + let layout = tcx.layout_of(ty::ParamEnv::reveal_all().and(ty)).unwrap(); + Self::from_scalar(Scalar::from_i8(c as i8), layout) + } + #[inline] pub fn to_const_int(self) -> ConstInt { assert!(self.layout.ty.is_integral()); diff --git a/compiler/rustc_const_eval/src/interpret/operator.rs b/compiler/rustc_const_eval/src/interpret/operator.rs index 475c533077af5..5665bb4999f59 100644 --- a/compiler/rustc_const_eval/src/interpret/operator.rs +++ b/compiler/rustc_const_eval/src/interpret/operator.rs @@ -61,6 +61,11 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { } impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { + fn three_way_compare(&self, lhs: T, rhs: T) -> (ImmTy<'tcx, M::Provenance>, bool) { + let res = Ord::cmp(&lhs, &rhs); + return (ImmTy::from_ordering(res, *self.tcx), false); + } + fn binary_char_op( &self, bin_op: mir::BinOp, @@ -69,6 +74,10 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { ) -> (ImmTy<'tcx, M::Provenance>, bool) { use rustc_middle::mir::BinOp::*; + if bin_op == Cmp { + return self.three_way_compare(l, r); + } + let res = match bin_op { Eq => l == r, Ne => l != r, @@ -231,6 +240,11 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { let r = self.sign_extend(r, right_layout) as i128; return Ok((ImmTy::from_bool(op(&l, &r), *self.tcx), false)); } + if bin_op == Cmp { + let l = self.sign_extend(l, left_layout) as i128; + let r = self.sign_extend(r, right_layout) as i128; + return Ok(self.three_way_compare(l, r)); + } let op: Option (i128, bool)> = match bin_op { Div if r == 0 => throw_ub!(DivisionByZero), Rem if r == 0 => throw_ub!(RemainderByZero), @@ -270,6 +284,10 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { } } + if bin_op == Cmp { + return Ok(self.three_way_compare(l, r)); + } + let val = match bin_op { Eq => ImmTy::from_bool(l == r, *self.tcx), Ne => ImmTy::from_bool(l != r, *self.tcx), diff --git a/compiler/rustc_const_eval/src/transform/validate.rs b/compiler/rustc_const_eval/src/transform/validate.rs index 1664475f52a13..6cde2296459e5 100644 --- a/compiler/rustc_const_eval/src/transform/validate.rs +++ b/compiler/rustc_const_eval/src/transform/validate.rs @@ -987,6 +987,15 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { ) } } + Cmp => { + for x in [a, b] { + check_kinds!( + x, + "Cannot three-way compare non-integer type {:?}", + ty::Char | ty::Uint(..) | ty::Int(..) + ) + } + } AddUnchecked | SubUnchecked | MulUnchecked | Shl | ShlUnchecked | Shr | ShrUnchecked => { for x in [a, b] { diff --git a/compiler/rustc_const_eval/src/util/mod.rs b/compiler/rustc_const_eval/src/util/mod.rs index a8060463b69ed..0c3b59a0e78ec 100644 --- a/compiler/rustc_const_eval/src/util/mod.rs +++ b/compiler/rustc_const_eval/src/util/mod.rs @@ -19,7 +19,7 @@ pub fn binop_left_homogeneous(op: mir::BinOp) -> bool { match op { Add | AddUnchecked | Sub | SubUnchecked | Mul | MulUnchecked | Div | Rem | BitXor | BitAnd | BitOr | Offset | Shl | ShlUnchecked | Shr | ShrUnchecked => true, - Eq | Ne | Lt | Le | Gt | Ge => false, + Eq | Ne | Lt | Le | Gt | Ge | Cmp => false, } } @@ -30,7 +30,7 @@ pub fn binop_right_homogeneous(op: mir::BinOp) -> bool { use rustc_middle::mir::BinOp::*; match op { Add | AddUnchecked | Sub | SubUnchecked | Mul | MulUnchecked | Div | Rem | BitXor - | BitAnd | BitOr | Eq | Ne | Lt | Le | Gt | Ge => true, + | BitAnd | BitOr | Eq | Ne | Lt | Le | Gt | Ge | Cmp => true, Offset | Shl | ShlUnchecked | Shr | ShrUnchecked => false, } } diff --git a/compiler/rustc_hir/src/lang_items.rs b/compiler/rustc_hir/src/lang_items.rs index dbf86f5cf747d..ba971958aa92b 100644 --- a/compiler/rustc_hir/src/lang_items.rs +++ b/compiler/rustc_hir/src/lang_items.rs @@ -225,6 +225,7 @@ language_item_table! { Unpin, sym::unpin, unpin_trait, Target::Trait, GenericRequirement::None; Pin, sym::pin, pin_type, Target::Struct, GenericRequirement::None; + OrderingEnum, sym::Ordering, ordering_enum, Target::Enum, GenericRequirement::Exact(0); PartialEq, sym::eq, eq_trait, Target::Trait, GenericRequirement::Exact(1); PartialOrd, sym::partial_ord, partial_ord_trait, Target::Trait, GenericRequirement::Exact(1); CVoid, sym::c_void, c_void, Target::Enum, GenericRequirement::None; diff --git a/compiler/rustc_hir_analysis/src/check/intrinsic.rs b/compiler/rustc_hir_analysis/src/check/intrinsic.rs index f482ae4f5fa07..93d71e4bfb933 100644 --- a/compiler/rustc_hir_analysis/src/check/intrinsic.rs +++ b/compiler/rustc_hir_analysis/src/check/intrinsic.rs @@ -107,6 +107,7 @@ pub fn intrinsic_operation_unsafety(tcx: TyCtxt<'_>, intrinsic_id: LocalDefId) - | sym::cttz | sym::bswap | sym::bitreverse + | sym::three_way_compare | sym::discriminant_value | sym::type_id | sym::likely @@ -418,6 +419,10 @@ pub fn check_intrinsic_type( | sym::bswap | sym::bitreverse => (1, 0, vec![param(0)], param(0)), + sym::three_way_compare => { + (1, 0, vec![param(0), param(0)], tcx.ty_ordering_enum(Some(span))) + } + sym::add_with_overflow | sym::sub_with_overflow | sym::mul_with_overflow => { (1, 0, vec![param(0), param(0)], Ty::new_tup(tcx, &[param(0), tcx.types.bool])) } diff --git a/compiler/rustc_middle/src/mir/syntax.rs b/compiler/rustc_middle/src/mir/syntax.rs index 36b7a48b2a2b8..a33de2efe2fad 100644 --- a/compiler/rustc_middle/src/mir/syntax.rs +++ b/compiler/rustc_middle/src/mir/syntax.rs @@ -1444,6 +1444,8 @@ pub enum BinOp { Ge, /// The `>` operator (greater than) Gt, + /// The `<=>` operator (three-way comparison, like `Ord::cmp`) + Cmp, /// The `ptr.offset` operator Offset, } diff --git a/compiler/rustc_middle/src/mir/tcx.rs b/compiler/rustc_middle/src/mir/tcx.rs index 56a0a62339700..74c2aef8fcc20 100644 --- a/compiler/rustc_middle/src/mir/tcx.rs +++ b/compiler/rustc_middle/src/mir/tcx.rs @@ -276,6 +276,11 @@ impl<'tcx> BinOp { &BinOp::Eq | &BinOp::Lt | &BinOp::Le | &BinOp::Ne | &BinOp::Ge | &BinOp::Gt => { tcx.types.bool } + &BinOp::Cmp => { + // these should be integer-like types of the same size. + assert_eq!(lhs_ty, rhs_ty); + tcx.ty_ordering_enum(None) + } } } } @@ -312,7 +317,8 @@ impl BinOp { BinOp::Gt => hir::BinOpKind::Gt, BinOp::Le => hir::BinOpKind::Le, BinOp::Ge => hir::BinOpKind::Ge, - BinOp::AddUnchecked + BinOp::Cmp + | BinOp::AddUnchecked | BinOp::SubUnchecked | BinOp::MulUnchecked | BinOp::ShlUnchecked diff --git a/compiler/rustc_middle/src/ty/context.rs b/compiler/rustc_middle/src/ty/context.rs index 3393f44484388..01fd815be6d67 100644 --- a/compiler/rustc_middle/src/ty/context.rs +++ b/compiler/rustc_middle/src/ty/context.rs @@ -905,6 +905,13 @@ impl<'tcx> TyCtxt<'tcx> { self.get_lang_items(()) } + /// Gets a `Ty` representing the [`LangItem::OrderingEnum`] + #[track_caller] + pub fn ty_ordering_enum(self, span: Option) -> Ty<'tcx> { + let ordering_enum = self.require_lang_item(hir::LangItem::OrderingEnum, span); + self.type_of(ordering_enum).no_bound_vars().unwrap() + } + /// Obtain the given diagnostic item's `DefId`. Use `is_diagnostic_item` if you just want to /// compare against another `DefId`, since `is_diagnostic_item` is cheaper. pub fn get_diagnostic_item(self, name: Symbol) -> Option { diff --git a/compiler/rustc_mir_transform/src/lower_intrinsics.rs b/compiler/rustc_mir_transform/src/lower_intrinsics.rs index 7d4c1b9c21a62..7e8920604c176 100644 --- a/compiler/rustc_mir_transform/src/lower_intrinsics.rs +++ b/compiler/rustc_mir_transform/src/lower_intrinsics.rs @@ -90,6 +90,7 @@ impl<'tcx> MirPass<'tcx> for LowerIntrinsics { sym::wrapping_add | sym::wrapping_sub | sym::wrapping_mul + | sym::three_way_compare | sym::unchecked_add | sym::unchecked_sub | sym::unchecked_mul @@ -109,6 +110,7 @@ impl<'tcx> MirPass<'tcx> for LowerIntrinsics { sym::wrapping_add => BinOp::Add, sym::wrapping_sub => BinOp::Sub, sym::wrapping_mul => BinOp::Mul, + sym::three_way_compare => BinOp::Cmp, sym::unchecked_add => BinOp::AddUnchecked, sym::unchecked_sub => BinOp::SubUnchecked, sym::unchecked_mul => BinOp::MulUnchecked, diff --git a/compiler/rustc_mir_transform/src/promote_consts.rs b/compiler/rustc_mir_transform/src/promote_consts.rs index 2951897ebd697..d64cb9b12b2bd 100644 --- a/compiler/rustc_mir_transform/src/promote_consts.rs +++ b/compiler/rustc_mir_transform/src/promote_consts.rs @@ -525,6 +525,7 @@ impl<'tcx> Validator<'_, 'tcx> { | BinOp::Lt | BinOp::Ge | BinOp::Gt + | BinOp::Cmp | BinOp::Offset | BinOp::Add | BinOp::AddUnchecked diff --git a/compiler/rustc_smir/src/rustc_smir/convert/mir.rs b/compiler/rustc_smir/src/rustc_smir/convert/mir.rs index b6a722da602e0..22e7e5323c776 100644 --- a/compiler/rustc_smir/src/rustc_smir/convert/mir.rs +++ b/compiler/rustc_smir/src/rustc_smir/convert/mir.rs @@ -493,6 +493,7 @@ impl<'tcx> Stable<'tcx> for mir::BinOp { BinOp::Ne => stable_mir::mir::BinOp::Ne, BinOp::Ge => stable_mir::mir::BinOp::Ge, BinOp::Gt => stable_mir::mir::BinOp::Gt, + BinOp::Cmp => stable_mir::mir::BinOp::Cmp, BinOp::Offset => stable_mir::mir::BinOp::Offset, } } diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs index 73fcd2a76dfc0..7dffeab992284 100644 --- a/compiler/rustc_span/src/symbol.rs +++ b/compiler/rustc_span/src/symbol.rs @@ -1785,6 +1785,7 @@ symbols! { thread, thread_local, thread_local_macro, + three_way_compare, thumb2, thumb_mode: "thumb-mode", tmm_reg, diff --git a/compiler/rustc_ty_utils/src/consts.rs b/compiler/rustc_ty_utils/src/consts.rs index 2a2e53a81ed81..acbcc3918b2fb 100644 --- a/compiler/rustc_ty_utils/src/consts.rs +++ b/compiler/rustc_ty_utils/src/consts.rs @@ -82,7 +82,7 @@ fn check_binop(op: mir::BinOp) -> bool { match op { Add | AddUnchecked | Sub | SubUnchecked | Mul | MulUnchecked | Div | Rem | BitXor | BitAnd | BitOr | Shl | ShlUnchecked | Shr | ShrUnchecked | Eq | Lt | Le | Ne | Ge - | Gt => true, + | Gt | Cmp => true, Offset => false, } } diff --git a/compiler/stable_mir/src/mir/body.rs b/compiler/stable_mir/src/mir/body.rs index 7c536a3e91402..636bf919f5c63 100644 --- a/compiler/stable_mir/src/mir/body.rs +++ b/compiler/stable_mir/src/mir/body.rs @@ -329,6 +329,7 @@ pub enum BinOp { Ne, Ge, Gt, + Cmp, Offset, } @@ -368,6 +369,9 @@ impl BinOp { assert!(lhs_kind.is_primitive() || lhs_kind.is_raw_ptr() || lhs_kind.is_fn_ptr()); Ty::bool_ty() } + BinOp::Cmp => { + unimplemented!("Should cmp::Ordering be a RigidTy?"); + } } } } diff --git a/library/core/src/cmp.rs b/library/core/src/cmp.rs index a2f07814726ac..7581c6dd759c3 100644 --- a/library/core/src/cmp.rs +++ b/library/core/src/cmp.rs @@ -376,6 +376,7 @@ pub struct AssertParamIsEq { /// ``` #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug, Hash)] #[stable(feature = "rust1", since = "1.0.0")] +#[cfg_attr(not(bootstrap), lang = "Ordering")] #[repr(i8)] pub enum Ordering { /// An ordering where a compared value is less than another. @@ -1563,12 +1564,19 @@ mod impls { impl Ord for $t { #[inline] fn cmp(&self, other: &$t) -> Ordering { + #[cfg(bootstrap)] + { // The order here is important to generate more optimal assembly. // See for more info. if *self < *other { Less } else if *self == *other { Equal } else { Greater } } + #[cfg(not(bootstrap))] + { + crate::intrinsics::three_way_compare(*self, *other) + } + } } )*) } diff --git a/library/core/src/intrinsics.rs b/library/core/src/intrinsics.rs index dec31548fc8c3..af771b88dc30e 100644 --- a/library/core/src/intrinsics.rs +++ b/library/core/src/intrinsics.rs @@ -2146,6 +2146,18 @@ extern "rust-intrinsic" { #[rustc_nounwind] pub fn bitreverse(x: T) -> T; + /// Does a three-way comparison between the two integer arguments. + /// + /// This is included as an intrinsic as it's useful to let it be one thing + /// in MIR, rather than the multiple checks and switches that make its IR + /// large and difficult to optimize. + /// + /// The stabilized version of this intrinsic is [`Ord::cmp`]. + #[cfg(not(bootstrap))] + #[rustc_const_unstable(feature = "const_three_way_compare", issue = "none")] + #[rustc_safe_intrinsic] + pub fn three_way_compare(lhs: T, rhs: T) -> crate::cmp::Ordering; + /// Performs checked integer addition. /// /// Note that, unlike most intrinsics, this is safe to call; diff --git a/library/core/tests/intrinsics.rs b/library/core/tests/intrinsics.rs index 740565d0df6b4..a9ef5415ada6f 100644 --- a/library/core/tests/intrinsics.rs +++ b/library/core/tests/intrinsics.rs @@ -99,3 +99,22 @@ fn test_const_deallocate_at_runtime() { const_deallocate(core::ptr::null_mut(), 1, 1); // nop } } + +#[cfg(not(bootstrap))] +#[test] +fn test_three_way_compare_in_const_contexts() { + use core::cmp::Ordering::*; + use core::intrinsics::three_way_compare; + + const { + assert!(Less as i8 == three_way_compare(123_u16, 456) as _); + assert!(Equal as i8 == three_way_compare(456_u16, 456) as _); + assert!(Greater as i8 == three_way_compare(789_u16, 456) as _); + assert!(Less as i8 == three_way_compare('A', 'B') as _); + assert!(Equal as i8 == three_way_compare('B', 'B') as _); + assert!(Greater as i8 == three_way_compare('C', 'B') as _); + assert!(Less as i8 == three_way_compare(-123_i16, 456) as _); + assert!(Equal as i8 == three_way_compare(456_i16, 456) as _); + assert!(Greater as i8 == three_way_compare(123_i16, -456) as _); + } +} diff --git a/library/core/tests/lib.rs b/library/core/tests/lib.rs index 421062f5873cd..6a2ff3a2285a8 100644 --- a/library/core/tests/lib.rs +++ b/library/core/tests/lib.rs @@ -21,6 +21,7 @@ #![feature(const_pointer_is_aligned)] #![feature(const_ptr_as_ref)] #![feature(const_ptr_write)] +#![cfg_attr(not(bootstrap), feature(const_three_way_compare))] #![feature(const_trait_impl)] #![feature(const_likely)] #![feature(const_location_fields)] diff --git a/tests/assembly/x86_64-cmp.rs b/tests/assembly/x86_64-cmp.rs new file mode 100644 index 0000000000000..31efdda1bfafa --- /dev/null +++ b/tests/assembly/x86_64-cmp.rs @@ -0,0 +1,51 @@ +//@ revisions: DEBUG OPTIM +//@ [DEBUG] compile-flags: -C opt-level=0 +//@ [OPTIM] compile-flags: -C opt-level=3 +//@ assembly-output: emit-asm +//@ compile-flags: --crate-type=lib -C llvm-args=-x86-asm-syntax=intel +//@ only-x86_64 +//@ ignore-sgx + +#![feature(core_intrinsics)] + +use std::intrinsics::three_way_compare; + +#[no_mangle] +// CHECK-LABEL: signed_cmp: +pub fn signed_cmp(a: i16, b: i16) -> std::cmp::Ordering { + // DEBUG: cmp + // DEBUG: setg + // DEBUG: and + // DEBUG: cmp + // DEBUG: setl + // DEBUG: and + // DEBUG: sub + + // OPTIM: xor + // OPTIM: cmp + // OPTIM: setne + // OPTIM: mov + // OPTIM: cmovge + // OPTIM: ret + three_way_compare(a, b) +} + +#[no_mangle] +// CHECK-LABEL: unsigned_cmp: +pub fn unsigned_cmp(a: u16, b: u16) -> std::cmp::Ordering { + // DEBUG: cmp + // DEBUG: seta + // DEBUG: and + // DEBUG: cmp + // DEBUG: setb + // DEBUG: and + // DEBUG: sub + + // OPTIM: xor + // OPTIM: cmp + // OPTIM: setne + // OPTIM: mov + // OPTIM: cmovae + // OPTIM: ret + three_way_compare(a, b) +} diff --git a/tests/codegen/intrinsics/three_way_compare.rs b/tests/codegen/intrinsics/three_way_compare.rs new file mode 100644 index 0000000000000..f3b631abc227e --- /dev/null +++ b/tests/codegen/intrinsics/three_way_compare.rs @@ -0,0 +1,47 @@ +//@ revisions: DEBUG OPTIM +//@ [DEBUG] compile-flags: -C opt-level=0 +//@ [OPTIM] compile-flags: -C opt-level=3 +//@ compile-flags: -C no-prepopulate-passes + +#![crate_type = "lib"] +#![feature(core_intrinsics)] + +use std::intrinsics::three_way_compare; + +#[no_mangle] +// CHECK-LABEL: @signed_cmp +// DEBUG-SAME: (i16 %a, i16 %b) +// OPTIM-SAME: (i16 noundef %a, i16 noundef %b) +pub fn signed_cmp(a: i16, b: i16) -> std::cmp::Ordering { + // DEBUG: %[[GT:.+]] = icmp sgt i16 %a, %b + // DEBUG: %[[ZGT:.+]] = zext i1 %[[GT]] to i8 + // DEBUG: %[[LT:.+]] = icmp slt i16 %a, %b + // DEBUG: %[[ZLT:.+]] = zext i1 %[[LT]] to i8 + // DEBUG: %[[R:.+]] = sub nsw i8 %[[ZGT]], %[[ZLT]] + + // OPTIM: %[[LT:.+]] = icmp slt i16 %a, %b + // OPTIM: %[[NE:.+]] = icmp ne i16 %a, %b + // OPTIM: %[[CGE:.+]] = select i1 %[[NE]], i8 1, i8 0 + // OPTIM: %[[CGEL:.+]] = select i1 %[[LT]], i8 -1, i8 %[[CGE]] + // OPTIM: ret i8 %[[CGEL]] + three_way_compare(a, b) +} + +#[no_mangle] +// CHECK-LABEL: @unsigned_cmp +// DEBUG-SAME: (i16 %a, i16 %b) +// OPTIM-SAME: (i16 noundef %a, i16 noundef %b) +pub fn unsigned_cmp(a: u16, b: u16) -> std::cmp::Ordering { + // DEBUG: %[[GT:.+]] = icmp ugt i16 %a, %b + // DEBUG: %[[ZGT:.+]] = zext i1 %[[GT]] to i8 + // DEBUG: %[[LT:.+]] = icmp ult i16 %a, %b + // DEBUG: %[[ZLT:.+]] = zext i1 %[[LT]] to i8 + // DEBUG: %[[R:.+]] = sub nsw i8 %[[ZGT]], %[[ZLT]] + + // OPTIM: %[[LT:.+]] = icmp ult i16 %a, %b + // OPTIM: %[[NE:.+]] = icmp ne i16 %a, %b + // OPTIM: %[[CGE:.+]] = select i1 %[[NE]], i8 1, i8 0 + // OPTIM: %[[CGEL:.+]] = select i1 %[[LT]], i8 -1, i8 %[[CGE]] + // OPTIM: ret i8 %[[CGEL]] + three_way_compare(a, b) +} diff --git a/tests/mir-opt/lower_intrinsics.rs b/tests/mir-opt/lower_intrinsics.rs index 278ddfce1c331..e9eb392e79528 100644 --- a/tests/mir-opt/lower_intrinsics.rs +++ b/tests/mir-opt/lower_intrinsics.rs @@ -229,3 +229,18 @@ pub unsafe fn ptr_offset(p: *const i32, d: isize) -> *const i32 { core::intrinsics::offset(p, d) } + +// EMIT_MIR lower_intrinsics.three_way_compare_char.LowerIntrinsics.diff +pub fn three_way_compare_char(a: char, b: char) { + let _x = core::intrinsics::three_way_compare(a, b); +} + +// EMIT_MIR lower_intrinsics.three_way_compare_signed.LowerIntrinsics.diff +pub fn three_way_compare_signed(a: i16, b: i16) { + core::intrinsics::three_way_compare(a, b); +} + +// EMIT_MIR lower_intrinsics.three_way_compare_unsigned.LowerIntrinsics.diff +pub fn three_way_compare_unsigned(a: u32, b: u32) { + let _x = core::intrinsics::three_way_compare(a, b); +} diff --git a/tests/mir-opt/lower_intrinsics.three_way_compare_char.LowerIntrinsics.panic-abort.diff b/tests/mir-opt/lower_intrinsics.three_way_compare_char.LowerIntrinsics.panic-abort.diff new file mode 100644 index 0000000000000..816d620971588 --- /dev/null +++ b/tests/mir-opt/lower_intrinsics.three_way_compare_char.LowerIntrinsics.panic-abort.diff @@ -0,0 +1,34 @@ +- // MIR for `three_way_compare_char` before LowerIntrinsics ++ // MIR for `three_way_compare_char` after LowerIntrinsics + + fn three_way_compare_char(_1: char, _2: char) -> () { + debug a => _1; + debug b => _2; + let mut _0: (); + let _3: std::cmp::Ordering; + let mut _4: char; + let mut _5: char; + scope 1 { + debug _x => _3; + } + + bb0: { + StorageLive(_3); + StorageLive(_4); + _4 = _1; + StorageLive(_5); + _5 = _2; +- _3 = three_way_compare::(move _4, move _5) -> [return: bb1, unwind unreachable]; ++ _3 = Cmp(move _4, move _5); ++ goto -> bb1; + } + + bb1: { + StorageDead(_5); + StorageDead(_4); + _0 = const (); + StorageDead(_3); + return; + } + } + diff --git a/tests/mir-opt/lower_intrinsics.three_way_compare_char.LowerIntrinsics.panic-unwind.diff b/tests/mir-opt/lower_intrinsics.three_way_compare_char.LowerIntrinsics.panic-unwind.diff new file mode 100644 index 0000000000000..80b4bd7a2be75 --- /dev/null +++ b/tests/mir-opt/lower_intrinsics.three_way_compare_char.LowerIntrinsics.panic-unwind.diff @@ -0,0 +1,34 @@ +- // MIR for `three_way_compare_char` before LowerIntrinsics ++ // MIR for `three_way_compare_char` after LowerIntrinsics + + fn three_way_compare_char(_1: char, _2: char) -> () { + debug a => _1; + debug b => _2; + let mut _0: (); + let _3: std::cmp::Ordering; + let mut _4: char; + let mut _5: char; + scope 1 { + debug _x => _3; + } + + bb0: { + StorageLive(_3); + StorageLive(_4); + _4 = _1; + StorageLive(_5); + _5 = _2; +- _3 = three_way_compare::(move _4, move _5) -> [return: bb1, unwind continue]; ++ _3 = Cmp(move _4, move _5); ++ goto -> bb1; + } + + bb1: { + StorageDead(_5); + StorageDead(_4); + _0 = const (); + StorageDead(_3); + return; + } + } + diff --git a/tests/mir-opt/lower_intrinsics.three_way_compare_signed.LowerIntrinsics.panic-abort.diff b/tests/mir-opt/lower_intrinsics.three_way_compare_signed.LowerIntrinsics.panic-abort.diff new file mode 100644 index 0000000000000..05c20aaa09a27 --- /dev/null +++ b/tests/mir-opt/lower_intrinsics.three_way_compare_signed.LowerIntrinsics.panic-abort.diff @@ -0,0 +1,31 @@ +- // MIR for `three_way_compare_signed` before LowerIntrinsics ++ // MIR for `three_way_compare_signed` after LowerIntrinsics + + fn three_way_compare_signed(_1: i16, _2: i16) -> () { + debug a => _1; + debug b => _2; + let mut _0: (); + let _3: std::cmp::Ordering; + let mut _4: i16; + let mut _5: i16; + + bb0: { + StorageLive(_3); + StorageLive(_4); + _4 = _1; + StorageLive(_5); + _5 = _2; +- _3 = three_way_compare::(move _4, move _5) -> [return: bb1, unwind unreachable]; ++ _3 = Cmp(move _4, move _5); ++ goto -> bb1; + } + + bb1: { + StorageDead(_5); + StorageDead(_4); + StorageDead(_3); + _0 = const (); + return; + } + } + diff --git a/tests/mir-opt/lower_intrinsics.three_way_compare_signed.LowerIntrinsics.panic-unwind.diff b/tests/mir-opt/lower_intrinsics.three_way_compare_signed.LowerIntrinsics.panic-unwind.diff new file mode 100644 index 0000000000000..8a254d02a47b2 --- /dev/null +++ b/tests/mir-opt/lower_intrinsics.three_way_compare_signed.LowerIntrinsics.panic-unwind.diff @@ -0,0 +1,31 @@ +- // MIR for `three_way_compare_signed` before LowerIntrinsics ++ // MIR for `three_way_compare_signed` after LowerIntrinsics + + fn three_way_compare_signed(_1: i16, _2: i16) -> () { + debug a => _1; + debug b => _2; + let mut _0: (); + let _3: std::cmp::Ordering; + let mut _4: i16; + let mut _5: i16; + + bb0: { + StorageLive(_3); + StorageLive(_4); + _4 = _1; + StorageLive(_5); + _5 = _2; +- _3 = three_way_compare::(move _4, move _5) -> [return: bb1, unwind continue]; ++ _3 = Cmp(move _4, move _5); ++ goto -> bb1; + } + + bb1: { + StorageDead(_5); + StorageDead(_4); + StorageDead(_3); + _0 = const (); + return; + } + } + diff --git a/tests/mir-opt/lower_intrinsics.three_way_compare_unsigned.LowerIntrinsics.panic-abort.diff b/tests/mir-opt/lower_intrinsics.three_way_compare_unsigned.LowerIntrinsics.panic-abort.diff new file mode 100644 index 0000000000000..437614ec6738d --- /dev/null +++ b/tests/mir-opt/lower_intrinsics.three_way_compare_unsigned.LowerIntrinsics.panic-abort.diff @@ -0,0 +1,34 @@ +- // MIR for `three_way_compare_unsigned` before LowerIntrinsics ++ // MIR for `three_way_compare_unsigned` after LowerIntrinsics + + fn three_way_compare_unsigned(_1: u32, _2: u32) -> () { + debug a => _1; + debug b => _2; + let mut _0: (); + let _3: std::cmp::Ordering; + let mut _4: u32; + let mut _5: u32; + scope 1 { + debug _x => _3; + } + + bb0: { + StorageLive(_3); + StorageLive(_4); + _4 = _1; + StorageLive(_5); + _5 = _2; +- _3 = three_way_compare::(move _4, move _5) -> [return: bb1, unwind unreachable]; ++ _3 = Cmp(move _4, move _5); ++ goto -> bb1; + } + + bb1: { + StorageDead(_5); + StorageDead(_4); + _0 = const (); + StorageDead(_3); + return; + } + } + diff --git a/tests/mir-opt/lower_intrinsics.three_way_compare_unsigned.LowerIntrinsics.panic-unwind.diff b/tests/mir-opt/lower_intrinsics.three_way_compare_unsigned.LowerIntrinsics.panic-unwind.diff new file mode 100644 index 0000000000000..7d6137979c81b --- /dev/null +++ b/tests/mir-opt/lower_intrinsics.three_way_compare_unsigned.LowerIntrinsics.panic-unwind.diff @@ -0,0 +1,34 @@ +- // MIR for `three_way_compare_unsigned` before LowerIntrinsics ++ // MIR for `three_way_compare_unsigned` after LowerIntrinsics + + fn three_way_compare_unsigned(_1: u32, _2: u32) -> () { + debug a => _1; + debug b => _2; + let mut _0: (); + let _3: std::cmp::Ordering; + let mut _4: u32; + let mut _5: u32; + scope 1 { + debug _x => _3; + } + + bb0: { + StorageLive(_3); + StorageLive(_4); + _4 = _1; + StorageLive(_5); + _5 = _2; +- _3 = three_way_compare::(move _4, move _5) -> [return: bb1, unwind continue]; ++ _3 = Cmp(move _4, move _5); ++ goto -> bb1; + } + + bb1: { + StorageDead(_5); + StorageDead(_4); + _0 = const (); + StorageDead(_3); + return; + } + } + diff --git a/tests/mir-opt/pre-codegen/derived_ord.{impl#0}-partial_cmp.PreCodegen.after.mir b/tests/mir-opt/pre-codegen/derived_ord.{impl#0}-partial_cmp.PreCodegen.after.mir index dd7fce3ed0b46..7ce2d229d5474 100644 --- a/tests/mir-opt/pre-codegen/derived_ord.{impl#0}-partial_cmp.PreCodegen.after.mir +++ b/tests/mir-opt/pre-codegen/derived_ord.{impl#0}-partial_cmp.PreCodegen.after.mir @@ -6,37 +6,33 @@ fn ::partial_cmp(_1: &MultiField, _2: &M let mut _0: std::option::Option; let mut _3: &char; let mut _4: &char; - let mut _10: std::option::Option; + let mut _8: std::option::Option; + let mut _9: i8; + let mut _10: &i16; let mut _11: &i16; - let mut _12: &i16; - let _18: std::option::Option; scope 1 { - debug cmp => _18; + debug cmp => _8; } scope 2 (inlined std::cmp::impls::::partial_cmp) { debug self => _3; debug other => _4; - let mut _9: std::cmp::Ordering; + let mut _7: std::cmp::Ordering; scope 3 (inlined std::cmp::impls::::cmp) { debug self => _3; debug other => _4; let mut _5: char; let mut _6: char; - let mut _7: bool; - let mut _8: bool; } } scope 4 (inlined std::cmp::impls::::partial_cmp) { - debug self => _11; - debug other => _12; - let mut _17: std::cmp::Ordering; + debug self => _10; + debug other => _11; + let mut _14: std::cmp::Ordering; scope 5 (inlined std::cmp::impls::::cmp) { - debug self => _11; - debug other => _12; + debug self => _10; + debug other => _11; + let mut _12: i16; let mut _13: i16; - let mut _14: i16; - let mut _15: bool; - let mut _16: bool; } } @@ -45,115 +41,46 @@ fn ::partial_cmp(_1: &MultiField, _2: &M _3 = &((*_1).0: char); StorageLive(_4); _4 = &((*_2).0: char); - StorageLive(_9); StorageLive(_5); - StorageLive(_6); - StorageLive(_7); _5 = ((*_1).0: char); + StorageLive(_6); _6 = ((*_2).0: char); - _7 = Lt(_5, _6); - switchInt(move _7) -> [0: bb1, otherwise: bb10]; - } - - bb1: { - StorageLive(_8); - _8 = Eq(_5, _6); - switchInt(move _8) -> [0: bb2, otherwise: bb3]; - } - - bb2: { - _9 = const Greater; - StorageDead(_8); - StorageDead(_7); + _7 = Cmp(move _5, move _6); StorageDead(_6); StorageDead(_5); - _10 = Option::::Some(move _9); - StorageDead(_9); + _8 = Option::::Some(_7); StorageDead(_4); StorageDead(_3); - goto -> bb11; + _9 = discriminant(_7); + switchInt(move _9) -> [0: bb1, otherwise: bb2]; } - bb3: { - StorageDead(_8); - StorageDead(_7); - StorageDead(_6); - StorageDead(_5); - StorageDead(_9); - StorageDead(_4); - StorageDead(_3); + bb1: { + StorageLive(_10); + _10 = &((*_1).1: i16); StorageLive(_11); - _11 = &((*_1).1: i16); + _11 = &((*_2).1: i16); + StorageLive(_14); StorageLive(_12); - _12 = &((*_2).1: i16); - StorageLive(_17); + _12 = ((*_1).1: i16); StorageLive(_13); - StorageLive(_14); - StorageLive(_15); - _13 = ((*_1).1: i16); - _14 = ((*_2).1: i16); - _15 = Lt(_13, _14); - switchInt(move _15) -> [0: bb4, otherwise: bb8]; - } - - bb4: { - StorageLive(_16); - _16 = Eq(_13, _14); - switchInt(move _16) -> [0: bb5, otherwise: bb6]; - } - - bb5: { - _17 = const Greater; - goto -> bb7; - } - - bb6: { - _17 = const Equal; - goto -> bb7; - } - - bb7: { - StorageDead(_16); - goto -> bb9; - } - - bb8: { - _17 = const Less; - goto -> bb9; - } - - bb9: { - StorageDead(_15); - StorageDead(_14); + _13 = ((*_2).1: i16); + _14 = Cmp(move _12, move _13); StorageDead(_13); - _0 = Option::::Some(move _17); - StorageDead(_17); StorageDead(_12); + _0 = Option::::Some(move _14); + StorageDead(_14); StorageDead(_11); - goto -> bb12; + StorageDead(_10); + goto -> bb3; } - bb10: { - _9 = const Less; - StorageDead(_7); - StorageDead(_6); - StorageDead(_5); - _10 = Option::::Some(move _9); - StorageDead(_9); - StorageDead(_4); - StorageDead(_3); - goto -> bb11; - } - - bb11: { - StorageLive(_18); - _18 = _10; - _0 = _10; - StorageDead(_18); - goto -> bb12; + bb2: { + _0 = _8; + goto -> bb3; } - bb12: { + bb3: { return; } }