Skip to content

Commit

Permalink
Auto merge of #120594 - saethlin:delayed-debug-asserts, r=oli-obk
Browse files Browse the repository at this point in the history
Toggle assert_unsafe_precondition in codegen instead of expansion

The goal of this PR is to make some of the unsafe precondition checks in the standard library available in debug builds. Some UI tests are included to verify that it does that.

The diff is large, but most of it is blessing mir-opt tests and I've also split up this PR so it can be reviewed commit-by-commit.

This PR:
1. Adds a new intrinsic, `debug_assertions` which is lowered to a new MIR NullOp, and only to a constant after monomorphization
2. Rewrites `assume_unsafe_precondition` to check the new intrinsic, and be monomorphic.
3. Skips codegen of the `assume` intrinsic in unoptimized builds, because that was silly before but with these checks it's *very* silly
4. The checks with the most overhead are `ptr::read`/`ptr::write` and `NonNull::new_unchecked`. I've simply added `#[cfg(debug_assertions)]` to the checks for `ptr::read`/`ptr::write` because I was unable to come up with any (good) ideas for decreasing their impact. But for `NonNull::new_unchecked` I found that the majority of callers can use a different function, often a safe one.

Yes, this PR slows down the compile time of some programs. But in our benchmark suite it's never more than 1% icount, and the average icount change in debug-full programs is 0.22%. I think that is acceptable for such an improvement in developer experience.

#120539 (comment)
  • Loading branch information
bors committed Feb 9, 2024
2 parents 972452c + dbf817b commit 8fb67fb
Show file tree
Hide file tree
Showing 65 changed files with 1,439 additions and 1,252 deletions.
1 change: 1 addition & 0 deletions compiler/rustc_borrowck/src/type_check/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1983,6 +1983,7 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
ConstraintCategory::SizedBound,
);
}
&Rvalue::NullaryOp(NullOp::DebugAssertions, _) => {}

Rvalue::ShallowInitBox(operand, ty) => {
self.check_operand(operand, location);
Expand Down
9 changes: 9 additions & 0 deletions compiler/rustc_codegen_cranelift/src/base.rs
Original file line number Diff line number Diff line change
Expand Up @@ -767,6 +767,15 @@ fn codegen_stmt<'tcx>(
NullOp::OffsetOf(fields) => {
layout.offset_of_subfield(fx, fields.iter()).bytes()
}
NullOp::DebugAssertions => {
let val = fx.tcx.sess.opts.debug_assertions;
let val = CValue::by_val(
fx.bcx.ins().iconst(types::I8, i64::try_from(val).unwrap()),
fx.layout_of(fx.tcx.types.bool),
);
lval.write_cvalue(fx, val);
return;
}
};
let val = CValue::by_val(
fx.bcx.ins().iconst(fx.pointer_type, i64::try_from(val).unwrap()),
Expand Down
14 changes: 10 additions & 4 deletions compiler/rustc_codegen_ssa/src/mir/rvalue.rs
Original file line number Diff line number Diff line change
Expand Up @@ -672,17 +672,23 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
let val = match null_op {
mir::NullOp::SizeOf => {
assert!(bx.cx().type_is_sized(ty));
layout.size.bytes()
let val = layout.size.bytes();
bx.cx().const_usize(val)
}
mir::NullOp::AlignOf => {
assert!(bx.cx().type_is_sized(ty));
layout.align.abi.bytes()
let val = layout.align.abi.bytes();
bx.cx().const_usize(val)
}
mir::NullOp::OffsetOf(fields) => {
layout.offset_of_subfield(bx.cx(), fields.iter()).bytes()
let val = layout.offset_of_subfield(bx.cx(), fields.iter()).bytes();
bx.cx().const_usize(val)
}
mir::NullOp::DebugAssertions => {
let val = bx.tcx().sess.opts.debug_assertions;
bx.cx().const_bool(val)
}
};
let val = bx.cx().const_usize(val);
let tcx = self.cx.tcx();
OperandRef {
val: OperandValue::Immediate(val),
Expand Down
7 changes: 5 additions & 2 deletions compiler/rustc_codegen_ssa/src/mir/statement.rs
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
use rustc_middle::mir;
use rustc_middle::mir::NonDivergingIntrinsic;
use rustc_session::config::OptLevel;

use super::FunctionCx;
use super::LocalRef;
Expand Down Expand Up @@ -67,8 +68,10 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
self.codegen_coverage(bx, coverage, statement.source_info.scope);
}
mir::StatementKind::Intrinsic(box NonDivergingIntrinsic::Assume(ref op)) => {
let op_val = self.codegen_operand(bx, op);
bx.assume(op_val.immediate());
if !matches!(bx.tcx().sess.opts.optimize, OptLevel::No | OptLevel::Less) {
let op_val = self.codegen_operand(bx, op);
bx.assume(op_val.immediate());
}
}
mir::StatementKind::Intrinsic(box NonDivergingIntrinsic::CopyNonOverlapping(
mir::CopyNonOverlapping { ref count, ref src, ref dst },
Expand Down
20 changes: 16 additions & 4 deletions compiler/rustc_const_eval/src/interpret/step.rs
Original file line number Diff line number Diff line change
Expand Up @@ -246,13 +246,25 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
);
}
let val = match null_op {
mir::NullOp::SizeOf => layout.size.bytes(),
mir::NullOp::AlignOf => layout.align.abi.bytes(),
mir::NullOp::SizeOf => {
let val = layout.size.bytes();
Scalar::from_target_usize(val, self)
}
mir::NullOp::AlignOf => {
let val = layout.align.abi.bytes();
Scalar::from_target_usize(val, self)
}
mir::NullOp::OffsetOf(fields) => {
layout.offset_of_subfield(self, fields.iter()).bytes()
let val = layout.offset_of_subfield(self, fields.iter()).bytes();
Scalar::from_target_usize(val, self)
}
mir::NullOp::DebugAssertions => {
// The checks hidden behind this are always better done by the interpreter
// itself, because it knows the runtime state better.
Scalar::from_bool(false)
}
};
self.write_scalar(Scalar::from_target_usize(val, self), &dest)?;
self.write_scalar(val, &dest)?;
}

ShallowInitBox(ref operand, _) => {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -544,7 +544,10 @@ impl<'tcx> Visitor<'tcx> for Checker<'_, 'tcx> {

Rvalue::Cast(_, _, _) => {}

Rvalue::NullaryOp(NullOp::SizeOf | NullOp::AlignOf | NullOp::OffsetOf(_), _) => {}
Rvalue::NullaryOp(
NullOp::SizeOf | NullOp::AlignOf | NullOp::OffsetOf(_) | NullOp::DebugAssertions,
_,
) => {}
Rvalue::ShallowInitBox(_, _) => {}

Rvalue::UnaryOp(_, operand) => {
Expand Down
2 changes: 1 addition & 1 deletion compiler/rustc_const_eval/src/transform/validate.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1139,7 +1139,7 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
Rvalue::Repeat(_, _)
| Rvalue::ThreadLocalRef(_)
| Rvalue::AddressOf(_, _)
| Rvalue::NullaryOp(NullOp::SizeOf | NullOp::AlignOf, _)
| Rvalue::NullaryOp(NullOp::SizeOf | NullOp::AlignOf | NullOp::DebugAssertions, _)
| Rvalue::Discriminant(_) => {}
}
self.super_rvalue(rvalue, location);
Expand Down
5 changes: 4 additions & 1 deletion compiler/rustc_hir_analysis/src/check/intrinsic.rs
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,8 @@ pub fn intrinsic_operation_unsafety(tcx: TyCtxt<'_>, intrinsic_id: DefId) -> hir
| sym::forget
| sym::black_box
| sym::variant_count
| sym::ptr_mask => hir::Unsafety::Normal,
| sym::ptr_mask
| sym::debug_assertions => hir::Unsafety::Normal,
_ => hir::Unsafety::Unsafe,
};

Expand Down Expand Up @@ -461,6 +462,8 @@ pub fn check_intrinsic_type(tcx: TyCtxt<'_>, it: &hir::ForeignItem<'_>) {
(0, vec![Ty::new_imm_ptr(tcx, Ty::new_unit(tcx))], tcx.types.usize)
}

sym::debug_assertions => (0, Vec::new(), tcx.types.bool),

other => {
tcx.dcx().emit_err(UnrecognizedIntrinsicFunction { span: it.span, name: other });
return;
Expand Down
1 change: 1 addition & 0 deletions compiler/rustc_middle/src/mir/pretty.rs
Original file line number Diff line number Diff line change
Expand Up @@ -907,6 +907,7 @@ impl<'tcx> Debug for Rvalue<'tcx> {
NullOp::SizeOf => write!(fmt, "SizeOf({t})"),
NullOp::AlignOf => write!(fmt, "AlignOf({t})"),
NullOp::OffsetOf(fields) => write!(fmt, "OffsetOf({t}, {fields:?})"),
NullOp::DebugAssertions => write!(fmt, "cfg!(debug_assertions)"),
}
}
ThreadLocalRef(did) => ty::tls::with(|tcx| {
Expand Down
2 changes: 2 additions & 0 deletions compiler/rustc_middle/src/mir/syntax.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1361,6 +1361,8 @@ pub enum NullOp<'tcx> {
AlignOf,
/// Returns the offset of a field
OffsetOf(&'tcx List<(VariantIdx, FieldIdx)>),
/// cfg!(debug_assertions), but expanded in codegen
DebugAssertions,
}

#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
Expand Down
1 change: 1 addition & 0 deletions compiler/rustc_middle/src/mir/tcx.rs
Original file line number Diff line number Diff line change
Expand Up @@ -194,6 +194,7 @@ impl<'tcx> Rvalue<'tcx> {
Rvalue::NullaryOp(NullOp::SizeOf | NullOp::AlignOf | NullOp::OffsetOf(..), _) => {
tcx.types.usize
}
Rvalue::NullaryOp(NullOp::DebugAssertions, _) => tcx.types.bool,
Rvalue::Aggregate(ref ak, ref ops) => match **ak {
AggregateKind::Array(ty) => Ty::new_array(tcx, ty, ops.len() as u64),
AggregateKind::Tuple => {
Expand Down
5 changes: 4 additions & 1 deletion compiler/rustc_mir_dataflow/src/move_paths/builder.rs
Original file line number Diff line number Diff line change
Expand Up @@ -429,7 +429,10 @@ impl<'b, 'a, 'tcx, F: Fn(Ty<'tcx>) -> bool> Gatherer<'b, 'a, 'tcx, F> {
| Rvalue::AddressOf(..)
| Rvalue::Discriminant(..)
| Rvalue::Len(..)
| Rvalue::NullaryOp(NullOp::SizeOf | NullOp::AlignOf | NullOp::OffsetOf(..), _) => {}
| Rvalue::NullaryOp(
NullOp::SizeOf | NullOp::AlignOf | NullOp::OffsetOf(..) | NullOp::DebugAssertions,
_,
) => {}
}
}

Expand Down
1 change: 1 addition & 0 deletions compiler/rustc_mir_transform/src/const_prop_lint.rs
Original file line number Diff line number Diff line change
Expand Up @@ -634,6 +634,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> {
NullOp::OffsetOf(fields) => {
op_layout.offset_of_subfield(self, fields.iter()).bytes()
}
NullOp::DebugAssertions => return None,
};
ImmTy::from_scalar(Scalar::from_target_usize(val, self), layout).into()
}
Expand Down
1 change: 1 addition & 0 deletions compiler/rustc_mir_transform/src/gvn.rs
Original file line number Diff line number Diff line change
Expand Up @@ -489,6 +489,7 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
NullOp::OffsetOf(fields) => {
layout.offset_of_subfield(&self.ecx, fields.iter()).bytes()
}
NullOp::DebugAssertions => return None,
};
let usize_layout = self.ecx.layout_of(self.tcx.types.usize).unwrap();
let imm = ImmTy::try_from_uint(val, usize_layout)?;
Expand Down
25 changes: 25 additions & 0 deletions compiler/rustc_mir_transform/src/instsimplify.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,12 @@
use crate::simplify::simplify_duplicate_switch_targets;
use rustc_middle::mir::*;
use rustc_middle::ty::layout;
use rustc_middle::ty::layout::ValidityRequirement;
use rustc_middle::ty::{self, GenericArgsRef, ParamEnv, Ty, TyCtxt};
use rustc_span::symbol::Symbol;
use rustc_target::abi::FieldIdx;
use rustc_target::spec::abi::Abi;

pub struct InstSimplify;

Expand Down Expand Up @@ -38,6 +40,7 @@ impl<'tcx> MirPass<'tcx> for InstSimplify {
block.terminator.as_mut().unwrap(),
&mut block.statements,
);
ctx.simplify_nounwind_call(block.terminator.as_mut().unwrap());
simplify_duplicate_switch_targets(block.terminator.as_mut().unwrap());
}
}
Expand Down Expand Up @@ -252,6 +255,28 @@ impl<'tcx> InstSimplifyContext<'tcx, '_> {
terminator.kind = TerminatorKind::Goto { target: destination_block };
}

fn simplify_nounwind_call(&self, terminator: &mut Terminator<'tcx>) {
let TerminatorKind::Call { func, unwind, .. } = &mut terminator.kind else {
return;
};

let Some((def_id, _)) = func.const_fn_def() else {
return;
};

let body_ty = self.tcx.type_of(def_id).skip_binder();
let body_abi = match body_ty.kind() {
ty::FnDef(..) => body_ty.fn_sig(self.tcx).abi(),
ty::Closure(..) => Abi::RustCall,
ty::Coroutine(..) => Abi::Rust,
_ => bug!("unexpected body ty: {:?}", body_ty),
};

if !layout::fn_can_unwind(self.tcx, Some(def_id), body_abi) {
*unwind = UnwindAction::Unreachable;
}
}

fn simplify_intrinsic_assert(
&self,
terminator: &mut Terminator<'tcx>,
Expand Down
11 changes: 11 additions & 0 deletions compiler/rustc_mir_transform/src/lower_intrinsics.rs
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,17 @@ impl<'tcx> MirPass<'tcx> for LowerIntrinsics {
sym::unreachable => {
terminator.kind = TerminatorKind::Unreachable;
}
sym::debug_assertions => {
let target = target.unwrap();
block.statements.push(Statement {
source_info: terminator.source_info,
kind: StatementKind::Assign(Box::new((
*destination,
Rvalue::NullaryOp(NullOp::DebugAssertions, tcx.types.bool),
))),
});
terminator.kind = TerminatorKind::Goto { target };
}
sym::forget => {
if let Some(target) = *target {
block.statements.push(Statement {
Expand Down
1 change: 1 addition & 0 deletions compiler/rustc_mir_transform/src/promote_consts.rs
Original file line number Diff line number Diff line change
Expand Up @@ -446,6 +446,7 @@ impl<'tcx> Validator<'_, 'tcx> {
NullOp::SizeOf => {}
NullOp::AlignOf => {}
NullOp::OffsetOf(_) => {}
NullOp::DebugAssertions => {}
},

Rvalue::ShallowInitBox(_, _) => return Err(Unpromotable),
Expand Down
1 change: 1 addition & 0 deletions compiler/rustc_smir/src/rustc_smir/convert/mir.rs
Original file line number Diff line number Diff line change
Expand Up @@ -257,6 +257,7 @@ impl<'tcx> Stable<'tcx> for mir::NullOp<'tcx> {
OffsetOf(indices) => stable_mir::mir::NullOp::OffsetOf(
indices.iter().map(|idx| idx.stable(tables)).collect(),
),
DebugAssertions => stable_mir::mir::NullOp::DebugAssertions,
}
}
}
Expand Down
3 changes: 3 additions & 0 deletions compiler/stable_mir/src/mir/body.rs
Original file line number Diff line number Diff line change
Expand Up @@ -639,6 +639,7 @@ impl Rvalue {
Rvalue::NullaryOp(NullOp::SizeOf | NullOp::AlignOf | NullOp::OffsetOf(..), _) => {
Ok(Ty::usize_ty())
}
Rvalue::NullaryOp(NullOp::DebugAssertions, _) => Ok(Ty::bool_ty()),
Rvalue::Aggregate(ak, ops) => match *ak {
AggregateKind::Array(ty) => Ty::try_new_array(ty, ops.len() as u64),
AggregateKind::Tuple => Ok(Ty::new_tuple(
Expand Down Expand Up @@ -1005,6 +1006,8 @@ pub enum NullOp {
AlignOf,
/// Returns the offset of a field.
OffsetOf(Vec<(VariantIdx, FieldIdx)>),
/// cfg!(debug_assertions), but at codegen time
DebugAssertions,
}

impl Operand {
Expand Down
13 changes: 7 additions & 6 deletions library/alloc/src/raw_vec.rs
Original file line number Diff line number Diff line change
Expand Up @@ -207,11 +207,7 @@ impl<T, A: Allocator> RawVec<T, A> {
// Allocators currently return a `NonNull<[u8]>` whose length
// matches the size requested. If that ever changes, the capacity
// here should change to `ptr.len() / mem::size_of::<T>()`.
Self {
ptr: unsafe { Unique::new_unchecked(ptr.cast().as_ptr()) },
cap: unsafe { Cap(capacity) },
alloc,
}
Self { ptr: Unique::from(ptr.cast()), cap: unsafe { Cap(capacity) }, alloc }
}
}

Expand Down Expand Up @@ -239,6 +235,11 @@ impl<T, A: Allocator> RawVec<T, A> {
self.ptr.as_ptr()
}

#[inline]
pub fn non_null(&self) -> NonNull<T> {
NonNull::from(self.ptr)
}

/// Gets the capacity of the allocation.
///
/// This will always be `usize::MAX` if `T` is zero-sized.
Expand Down Expand Up @@ -398,7 +399,7 @@ impl<T, A: Allocator> RawVec<T, A> {
// Allocators currently return a `NonNull<[u8]>` whose length matches
// the size requested. If that ever changes, the capacity here should
// change to `ptr.len() / mem::size_of::<T>()`.
self.ptr = unsafe { Unique::new_unchecked(ptr.cast().as_ptr()) };
self.ptr = Unique::from(ptr.cast());
self.cap = unsafe { Cap(cap) };
}

Expand Down
2 changes: 1 addition & 1 deletion library/alloc/src/vec/into_iter.rs
Original file line number Diff line number Diff line change
Expand Up @@ -136,7 +136,7 @@ impl<T, A: Allocator> IntoIter<T, A> {
// struct and then overwriting &mut self.
// this creates less assembly
self.cap = 0;
self.buf = unsafe { NonNull::new_unchecked(RawVec::NEW.ptr()) };
self.buf = RawVec::NEW.non_null();
self.ptr = self.buf;
self.end = self.buf.as_ptr();

Expand Down
6 changes: 3 additions & 3 deletions library/alloc/src/vec/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2861,16 +2861,16 @@ impl<T, A: Allocator> IntoIterator for Vec<T, A> {
#[inline]
fn into_iter(self) -> Self::IntoIter {
unsafe {
let mut me = ManuallyDrop::new(self);
let me = ManuallyDrop::new(self);
let alloc = ManuallyDrop::new(ptr::read(me.allocator()));
let begin = me.as_mut_ptr();
let buf = me.buf.non_null();
let begin = buf.as_ptr();
let end = if T::IS_ZST {
begin.wrapping_byte_add(me.len())
} else {
begin.add(me.len()) as *const T
};
let cap = me.buf.capacity();
let buf = NonNull::new_unchecked(begin);
IntoIter { buf, phantom: PhantomData, cap, alloc, ptr: buf, end }
}
}
Expand Down
2 changes: 1 addition & 1 deletion library/core/src/char/convert.rs
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ pub(super) const unsafe fn from_u32_unchecked(i: u32) -> char {
unsafe {
assert_unsafe_precondition!(
"invalid value for `char`",
(i: u32) => char_try_from_u32(i).is_ok()
(i: u32 = i) => char_try_from_u32(i).is_ok()
);
transmute(i)
}
Expand Down
2 changes: 1 addition & 1 deletion library/core/src/hint.rs
Original file line number Diff line number Diff line change
Expand Up @@ -148,7 +148,7 @@ pub const unsafe fn assert_unchecked(cond: bool) {
unsafe {
intrinsics::assert_unsafe_precondition!(
"hint::assert_unchecked must never be called when the condition is false",
(cond: bool) => cond,
(cond: bool = cond) => cond,
);
crate::intrinsics::assume(cond);
}
Expand Down
Loading

0 comments on commit 8fb67fb

Please sign in to comment.