diff --git a/compiler/rustc_mir_transform/src/cost_checker.rs b/compiler/rustc_mir_transform/src/cost_checker.rs index 7e401b5482f3d..efc6d35cbc3b3 100644 --- a/compiler/rustc_mir_transform/src/cost_checker.rs +++ b/compiler/rustc_mir_transform/src/cost_checker.rs @@ -9,6 +9,9 @@ const LANDINGPAD_PENALTY: usize = 50; const RESUME_PENALTY: usize = 45; const LARGE_SWITCH_PENALTY: usize = 20; const CONST_SWITCH_BONUS: usize = 10; +const MULTIPLE_MUT_PENALTY: usize = 50; +const REF_COPY_BONUS: usize = 5; +const MANY_PARAMETERS_BONUS: usize = 10; /// Verify that the callee body is compatible with the caller. #[derive(Clone)] @@ -31,6 +34,75 @@ impl<'b, 'tcx> CostChecker<'b, 'tcx> { CostChecker { tcx, param_env, callee_body, instance, penalty: 0, bonus: 0 } } + /// Add function-level costs not well-represented by the block-level costs. + /// + /// Needed because the `CostChecker` is used sometimes for just blocks, + /// and even the full `Inline` doesn't call `visit_body`, so there's nowhere + /// to put this logic in the visitor. + pub fn add_function_level_costs(&mut self) { + // There's usually extra stack costs to passing lots of parameters, + // so we'd rather inline that if the method isn't actually complex, + // especially for trait impls that ignore some parameters. + if self.callee_body.arg_count > 2 { + self.bonus += MANY_PARAMETERS_BONUS; + } + + fn is_call_like(bbd: &BasicBlockData<'_>) -> bool { + use TerminatorKind::*; + match bbd.terminator().kind { + Call { .. } + | Drop { .. } + | Assert { .. } + | Yield { .. } + | CoroutineDrop + | InlineAsm { .. } => true, + + Goto { .. } + | SwitchInt { .. } + | UnwindResume + | UnwindTerminate(_) + | Return + | Unreachable => false, + + FalseEdge { .. } | FalseUnwind { .. } => unreachable!(), + } + } + + // If the only has one Call (or similar), inlining isn't increasing the total + // number of calls, so give extra encouragement to inlining that. + if self.callee_body.basic_blocks.iter().filter(|bbd| is_call_like(bbd)).count() == 1 { + self.bonus += CALL_PENALTY; + } + + let callee_param_env = std::cell::LazyCell::new(|| { + let def_id = self.callee_body.source.def_id(); + self.tcx.param_env(def_id) + }); + + let mut mut_ref_count = 0; + for local in self.callee_body.args_iter() { + let ty = self.callee_body.local_decls[local].ty; + if let ty::Ref(_, referee, mtbl) = ty.kind() { + match mtbl { + Mutability::Mut => mut_ref_count += 1, + Mutability::Not => { + // Encourage inlining `&impl Copy` parameters, + // as that often lets us remove the indirection. + if referee.is_copy_modulo_regions(self.tcx, *callee_param_env) { + self.bonus += REF_COPY_BONUS; + } + } + } + } + } + + // Discourage inlining something with multiple `&mut` parameters, + // as the MIR inliner doesn't know how to preserve `noalias`. + if mut_ref_count > 1 { + self.penalty += MULTIPLE_MUT_PENALTY; + } + } + pub fn cost(&self) -> usize { usize::saturating_sub(self.penalty, self.bonus) } diff --git a/compiler/rustc_mir_transform/src/inline.rs b/compiler/rustc_mir_transform/src/inline.rs index 07482d0571a2f..a3336a5ef0f3c 100644 --- a/compiler/rustc_mir_transform/src/inline.rs +++ b/compiler/rustc_mir_transform/src/inline.rs @@ -85,13 +85,18 @@ fn inline<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) -> bool { } let param_env = tcx.param_env_reveal_all_normalized(def_id); + let codegen_fn_attrs = tcx.codegen_fn_attrs(def_id); let mut this = Inliner { tcx, param_env, - codegen_fn_attrs: tcx.codegen_fn_attrs(def_id), + codegen_fn_attrs, history: Vec::new(), changed: false, + caller_is_inline_shim: matches!( + codegen_fn_attrs.inline, + InlineAttr::Hint | InlineAttr::Always + ) && body_is_shim(body), }; let blocks = START_BLOCK..body.basic_blocks.next_index(); this.process_blocks(body, blocks); @@ -111,6 +116,7 @@ struct Inliner<'tcx> { history: Vec, /// Indicates that the caller body has been modified. changed: bool, + caller_is_inline_shim: bool, } impl<'tcx> Inliner<'tcx> { @@ -485,7 +491,9 @@ impl<'tcx> Inliner<'tcx> { ) -> Result<(), &'static str> { let tcx = self.tcx; - let mut threshold = if cross_crate_inlinable { + let mut threshold = if self.caller_is_inline_shim { + self.tcx.sess.opts.unstable_opts.inline_mir_shim_threshold.unwrap_or(30) + } else if cross_crate_inlinable { self.tcx.sess.opts.unstable_opts.inline_mir_hint_threshold.unwrap_or(100) } else { self.tcx.sess.opts.unstable_opts.inline_mir_threshold.unwrap_or(50) @@ -504,6 +512,8 @@ impl<'tcx> Inliner<'tcx> { let mut checker = CostChecker::new(self.tcx, self.param_env, Some(callsite.callee), callee_body); + checker.add_function_level_costs(); + // Traverse the MIR manually so we can account for the effects of inlining on the CFG. let mut work_list = vec![START_BLOCK]; let mut visited = BitSet::new_empty(callee_body.basic_blocks.len()); @@ -1091,3 +1101,37 @@ fn try_instance_mir<'tcx>( } Ok(tcx.instance_mir(instance)) } + +fn body_is_shim(body: &Body<'_>) -> bool { + let TerminatorKind::Call { target, .. } = body.basic_blocks[START_BLOCK].terminator().kind + else { + return false; + }; + if let Some(target) = target { + let TerminatorKind::Return = body.basic_blocks[target].terminator().kind else { + return false; + }; + } + + let max_blocks = if !body.is_polymorphic { + 2 + } else if target.is_none() { + 3 + } else { + 4 + }; + if body.basic_blocks.len() > max_blocks { + return false; + } + + body.basic_blocks.iter_enumerated().all(|(bb, bb_data)| { + bb == START_BLOCK + || matches!( + bb_data.terminator().kind, + TerminatorKind::Return + | TerminatorKind::Drop { .. } + | TerminatorKind::UnwindResume + | TerminatorKind::UnwindTerminate(_) + ) + }) +} diff --git a/compiler/rustc_session/src/options.rs b/compiler/rustc_session/src/options.rs index 80f7ca544f3c5..7bde471254067 100644 --- a/compiler/rustc_session/src/options.rs +++ b/compiler/rustc_session/src/options.rs @@ -1771,6 +1771,8 @@ options! { inline_mir_preserve_debug: Option = (None, parse_opt_bool, [TRACKED], "when MIR inlining, whether to preserve debug info for callee variables \ (default: preserve for debuginfo != None, otherwise remove)"), + inline_mir_shim_threshold: Option = (None, parse_opt_number, [TRACKED], + "a default MIR inlining threshold (default: 30)"), inline_mir_threshold: Option = (None, parse_opt_number, [TRACKED], "a default MIR inlining threshold (default: 50)"), input_stats: bool = (false, parse_bool, [UNTRACKED], diff --git a/tests/mir-opt/inline/inline_more_in_non_inline.marked_inline_direct.Inline.after.panic-abort.mir b/tests/mir-opt/inline/inline_more_in_non_inline.marked_inline_direct.Inline.after.panic-abort.mir new file mode 100644 index 0000000000000..522f772c6f48e --- /dev/null +++ b/tests/mir-opt/inline/inline_more_in_non_inline.marked_inline_direct.Inline.after.panic-abort.mir @@ -0,0 +1,22 @@ +// MIR for `marked_inline_direct` after Inline + +fn marked_inline_direct(_1: i32) -> () { + debug x => _1; + let mut _0: (); + let _2: (); + let mut _3: i32; + + bb0: { + StorageLive(_2); + StorageLive(_3); + _3 = _1; + _2 = call_twice(move _3) -> [return: bb1, unwind unreachable]; + } + + bb1: { + StorageDead(_3); + StorageDead(_2); + _0 = const (); + return; + } +} diff --git a/tests/mir-opt/inline/inline_more_in_non_inline.marked_inline_direct.Inline.after.panic-unwind.mir b/tests/mir-opt/inline/inline_more_in_non_inline.marked_inline_direct.Inline.after.panic-unwind.mir new file mode 100644 index 0000000000000..722b02eeba806 --- /dev/null +++ b/tests/mir-opt/inline/inline_more_in_non_inline.marked_inline_direct.Inline.after.panic-unwind.mir @@ -0,0 +1,22 @@ +// MIR for `marked_inline_direct` after Inline + +fn marked_inline_direct(_1: i32) -> () { + debug x => _1; + let mut _0: (); + let _2: (); + let mut _3: i32; + + bb0: { + StorageLive(_2); + StorageLive(_3); + _3 = _1; + _2 = call_twice(move _3) -> [return: bb1, unwind continue]; + } + + bb1: { + StorageDead(_3); + StorageDead(_2); + _0 = const (); + return; + } +} diff --git a/tests/mir-opt/inline/inline_more_in_non_inline.marked_inline_indirect.Inline.after.panic-abort.mir b/tests/mir-opt/inline/inline_more_in_non_inline.marked_inline_indirect.Inline.after.panic-abort.mir new file mode 100644 index 0000000000000..63b91cbce2d60 --- /dev/null +++ b/tests/mir-opt/inline/inline_more_in_non_inline.marked_inline_indirect.Inline.after.panic-abort.mir @@ -0,0 +1,27 @@ +// MIR for `marked_inline_indirect` after Inline + +fn marked_inline_indirect(_1: i32) -> () { + debug x => _1; + let mut _0: (); + let _2: (); + let mut _3: i32; + scope 1 (inlined marked_inline_direct) { + let _4: (); + } + + bb0: { + StorageLive(_2); + StorageLive(_3); + _3 = _1; + StorageLive(_4); + _4 = call_twice(move _3) -> [return: bb1, unwind unreachable]; + } + + bb1: { + StorageDead(_4); + StorageDead(_3); + StorageDead(_2); + _0 = const (); + return; + } +} diff --git a/tests/mir-opt/inline/inline_more_in_non_inline.marked_inline_indirect.Inline.after.panic-unwind.mir b/tests/mir-opt/inline/inline_more_in_non_inline.marked_inline_indirect.Inline.after.panic-unwind.mir new file mode 100644 index 0000000000000..7c84e98dd516c --- /dev/null +++ b/tests/mir-opt/inline/inline_more_in_non_inline.marked_inline_indirect.Inline.after.panic-unwind.mir @@ -0,0 +1,27 @@ +// MIR for `marked_inline_indirect` after Inline + +fn marked_inline_indirect(_1: i32) -> () { + debug x => _1; + let mut _0: (); + let _2: (); + let mut _3: i32; + scope 1 (inlined marked_inline_direct) { + let _4: (); + } + + bb0: { + StorageLive(_2); + StorageLive(_3); + _3 = _1; + StorageLive(_4); + _4 = call_twice(move _3) -> [return: bb1, unwind continue]; + } + + bb1: { + StorageDead(_4); + StorageDead(_3); + StorageDead(_2); + _0 = const (); + return; + } +} diff --git a/tests/mir-opt/inline/inline_more_in_non_inline.monomorphic_not_inline.Inline.after.panic-abort.mir b/tests/mir-opt/inline/inline_more_in_non_inline.monomorphic_not_inline.Inline.after.panic-abort.mir new file mode 100644 index 0000000000000..989014b8b476d --- /dev/null +++ b/tests/mir-opt/inline/inline_more_in_non_inline.monomorphic_not_inline.Inline.after.panic-abort.mir @@ -0,0 +1,34 @@ +// MIR for `monomorphic_not_inline` after Inline + +fn monomorphic_not_inline(_1: i32) -> () { + debug x => _1; + let mut _0: (); + let _2: (); + let mut _3: i32; + scope 1 (inlined call_twice) { + let _4: (); + let _5: (); + } + + bb0: { + StorageLive(_2); + StorageLive(_3); + _3 = _1; + StorageLive(_4); + StorageLive(_5); + _4 = other_thing(_3) -> [return: bb2, unwind unreachable]; + } + + bb1: { + StorageDead(_5); + StorageDead(_4); + StorageDead(_3); + StorageDead(_2); + _0 = const (); + return; + } + + bb2: { + _5 = other_thing(move _3) -> [return: bb1, unwind unreachable]; + } +} diff --git a/tests/mir-opt/inline/inline_more_in_non_inline.monomorphic_not_inline.Inline.after.panic-unwind.mir b/tests/mir-opt/inline/inline_more_in_non_inline.monomorphic_not_inline.Inline.after.panic-unwind.mir new file mode 100644 index 0000000000000..bd200719bbbe9 --- /dev/null +++ b/tests/mir-opt/inline/inline_more_in_non_inline.monomorphic_not_inline.Inline.after.panic-unwind.mir @@ -0,0 +1,34 @@ +// MIR for `monomorphic_not_inline` after Inline + +fn monomorphic_not_inline(_1: i32) -> () { + debug x => _1; + let mut _0: (); + let _2: (); + let mut _3: i32; + scope 1 (inlined call_twice) { + let _4: (); + let _5: (); + } + + bb0: { + StorageLive(_2); + StorageLive(_3); + _3 = _1; + StorageLive(_4); + StorageLive(_5); + _4 = other_thing(_3) -> [return: bb2, unwind continue]; + } + + bb1: { + StorageDead(_5); + StorageDead(_4); + StorageDead(_3); + StorageDead(_2); + _0 = const (); + return; + } + + bb2: { + _5 = other_thing(move _3) -> [return: bb1, unwind continue]; + } +} diff --git a/tests/mir-opt/inline/inline_more_in_non_inline.rs b/tests/mir-opt/inline/inline_more_in_non_inline.rs new file mode 100644 index 0000000000000..5968b97047096 --- /dev/null +++ b/tests/mir-opt/inline/inline_more_in_non_inline.rs @@ -0,0 +1,46 @@ +// EMIT_MIR_FOR_EACH_PANIC_STRATEGY +//@ compile-flags: -O --crate-type lib + +// To avoid MIR blow-up, don't inline large callees into simple shim callers, +// but *do* inline other trivial things. + +extern "Rust" { + fn other_thing(x: i32); +} + +#[inline] +unsafe fn call_twice(x: i32) { + unsafe { + other_thing(x); + other_thing(x); + } +} + +// EMIT_MIR inline_more_in_non_inline.monomorphic_not_inline.Inline.after.mir +#[no_mangle] +pub unsafe fn monomorphic_not_inline(x: i32) { + // CHECK-LABEL: monomorphic_not_inline + // CHECK: other_thing + // CHECK: other_thing + unsafe { call_twice(x) }; +} + +// EMIT_MIR inline_more_in_non_inline.marked_inline_direct.Inline.after.mir +#[inline] +pub unsafe fn marked_inline_direct(x: i32) { + // CHECK-LABEL: marked_inline_direct + // CHECK-NOT: other_thing + // CHECK: call_twice + // CHECK-NOT: other_thing + unsafe { call_twice(x) }; +} + +// EMIT_MIR inline_more_in_non_inline.marked_inline_indirect.Inline.after.mir +#[inline] +pub unsafe fn marked_inline_indirect(x: i32) { + // CHECK-LABEL: marked_inline_indirect + // CHECK-NOT: other_thing + // CHECK: call_twice + // CHECK-NOT: other_thing + unsafe { marked_inline_direct(x) }; +} diff --git a/tests/mir-opt/inline/inline_shims.drop.Inline.panic-abort.diff b/tests/mir-opt/inline/inline_shims.drop.Inline.panic-abort.diff index 45ce933a55ad2..2a36ccaab110a 100644 --- a/tests/mir-opt/inline/inline_shims.drop.Inline.panic-abort.diff +++ b/tests/mir-opt/inline/inline_shims.drop.Inline.panic-abort.diff @@ -11,30 +11,10 @@ + scope 1 (inlined std::ptr::drop_in_place::> - shim(Some(Vec))) { + let mut _6: &mut std::vec::Vec; + let mut _7: (); -+ scope 2 (inlined as Drop>::drop) { -+ let mut _8: *mut [A]; -+ let mut _9: *mut A; -+ let mut _10: usize; -+ scope 3 (inlined Vec::::as_mut_ptr) { -+ let mut _11: &alloc::raw_vec::RawVec; -+ scope 4 (inlined alloc::raw_vec::RawVec::::ptr) { -+ let mut _13: std::ptr::NonNull; -+ scope 5 (inlined Unique::::as_ptr) { -+ scope 6 (inlined NonNull::::as_ptr) { -+ let mut _12: *const A; -+ } -+ } -+ } -+ } -+ scope 7 (inlined slice_from_raw_parts_mut::) { -+ scope 8 (inlined std::ptr::from_raw_parts_mut::<[A], A>) { -+ } -+ } -+ } + } -+ scope 9 (inlined std::ptr::drop_in_place::> - shim(Some(Option))) { -+ let mut _14: isize; -+ let mut _15: isize; ++ scope 2 (inlined std::ptr::drop_in_place::> - shim(Some(Option))) { ++ let mut _8: isize; ++ let mut _9: isize; + } bb0: { @@ -45,24 +25,7 @@ + StorageLive(_6); + StorageLive(_7); + _6 = &mut (*_4); -+ StorageLive(_8); -+ StorageLive(_9); -+ StorageLive(_11); -+ _11 = &((*_6).0: alloc::raw_vec::RawVec); -+ StorageLive(_13); -+ _13 = ((((*_6).0: alloc::raw_vec::RawVec).0: std::ptr::Unique).0: std::ptr::NonNull); -+ StorageLive(_12); -+ _12 = (_13.0: *const A); -+ _9 = move _12 as *mut A (PtrToPtr); -+ StorageDead(_12); -+ StorageDead(_13); -+ StorageDead(_11); -+ StorageLive(_10); -+ _10 = ((*_6).1: usize); -+ _8 = *mut [A] from (_9, _10); -+ StorageDead(_10); -+ StorageDead(_9); -+ _7 = std::ptr::drop_in_place::<[A]>(move _8) -> [return: bb2, unwind unreachable]; ++ _7 = as Drop>::drop(move _6) -> [return: bb2, unwind unreachable]; } bb1: { @@ -73,20 +36,19 @@ StorageLive(_5); _5 = _2; - _0 = std::ptr::drop_in_place::>(move _5) -> [return: bb2, unwind unreachable]; -+ StorageLive(_14); -+ StorageLive(_15); -+ _14 = discriminant((*_5)); -+ switchInt(move _14) -> [0: bb3, otherwise: bb4]; ++ StorageLive(_8); ++ StorageLive(_9); ++ _8 = discriminant((*_5)); ++ switchInt(move _8) -> [0: bb3, otherwise: bb4]; } bb2: { -+ StorageDead(_8); + drop(((*_4).0: alloc::raw_vec::RawVec)) -> [return: bb1, unwind unreachable]; + } + + bb3: { -+ StorageDead(_15); -+ StorageDead(_14); ++ StorageDead(_9); ++ StorageDead(_8); StorageDead(_5); return; + } diff --git a/tests/mir-opt/pre-codegen/slice_iter.reverse_loop.PreCodegen.after.panic-abort.mir b/tests/mir-opt/pre-codegen/slice_iter.reverse_loop.PreCodegen.after.panic-abort.mir index 79c5bcfe9cd5c..fbb887fe76a59 100644 --- a/tests/mir-opt/pre-codegen/slice_iter.reverse_loop.PreCodegen.after.panic-abort.mir +++ b/tests/mir-opt/pre-codegen/slice_iter.reverse_loop.PreCodegen.after.panic-abort.mir @@ -7,90 +7,19 @@ fn reverse_loop(_1: &[T], _2: impl Fn(&T)) -> () { let mut _11: std::slice::Iter<'_, T>; let mut _12: std::iter::Rev>; let mut _13: std::iter::Rev>; - let mut _37: std::option::Option<&T>; - let mut _39: &impl Fn(&T); - let mut _40: (&T,); - let _41: (); + let mut _15: std::option::Option<&T>; + let mut _16: isize; + let mut _18: &impl Fn(&T); + let mut _19: (&T,); + let _20: (); scope 1 { debug iter => _13; - let _38: &T; + let _17: &T; scope 2 { - debug x => _38; + debug x => _17; } scope 17 (inlined > as Iterator>::next) { - scope 18 (inlined as DoubleEndedIterator>::next_back) { - let mut _14: *const *const T; - let mut _15: *const std::ptr::NonNull; - let mut _20: bool; - let mut _21: *const T; - let _36: &T; - scope 19 { - let _16: std::ptr::NonNull; - let _22: usize; - scope 20 { - } - scope 21 { - scope 25 (inlined as PartialEq>::eq) { - let mut _17: std::ptr::NonNull; - scope 26 (inlined NonNull::::as_ptr) { - let mut _18: *const T; - } - scope 27 (inlined NonNull::::as_ptr) { - let mut _19: *const T; - } - } - } - scope 22 (inlined std::ptr::const_ptr::::addr) { - scope 23 (inlined std::ptr::const_ptr::::cast::<()>) { - } - } - scope 24 (inlined std::ptr::const_ptr::::cast::>) { - } - } - scope 28 (inlined std::slice::Iter::<'_, T>::next_back_unchecked) { - let _29: std::ptr::NonNull; - scope 29 (inlined std::slice::Iter::<'_, T>::pre_dec_end) { - let mut _23: *mut *const T; - let mut _24: *mut std::ptr::NonNull; - let mut _25: std::ptr::NonNull; - let mut _28: std::ptr::NonNull; - let mut _30: *mut *const T; - let mut _31: *mut usize; - let mut _32: usize; - let mut _33: usize; - scope 30 { - scope 31 { - } - scope 32 { - scope 35 (inlined NonNull::::sub) { - scope 36 (inlined core::num::::unchecked_neg) { - scope 37 (inlined core::ub_checks::check_language_ub) { - scope 38 (inlined core::ub_checks::check_language_ub::runtime) { - } - } - } - scope 39 (inlined NonNull::::offset) { - let mut _26: *const T; - let mut _27: *const T; - } - } - } - scope 33 (inlined std::ptr::mut_ptr::::cast::) { - } - scope 34 (inlined std::ptr::mut_ptr::::cast::>) { - } - } - } - scope 40 (inlined NonNull::::as_ref::<'_>) { - let mut _34: std::ptr::NonNull; - scope 41 (inlined NonNull::::as_ptr) { - let mut _35: *const T; - } - scope 42 (inlined std::ptr::mut_ptr::::cast_const) { - } - } - } - } + let mut _14: &mut std::slice::Iter<'_, T>; } } scope 3 (inlined core::slice::::iter) { @@ -178,147 +107,45 @@ fn reverse_loop(_1: &[T], _2: impl Fn(&T)) -> () { } bb4: { - StorageLive(_37); - StorageLive(_22); - StorageLive(_21); - StorageLive(_16); - StorageLive(_36); - StorageLive(_20); - switchInt(const ::IS_ZST) -> [0: bb5, otherwise: bb6]; + StorageLive(_15); + StorageLive(_14); + _14 = &mut (_13.0: std::slice::Iter<'_, T>); + _15 = as DoubleEndedIterator>::next_back(move _14) -> [return: bb5, unwind unreachable]; } bb5: { - StorageLive(_15); - StorageLive(_14); - _14 = &raw const ((_13.0: std::slice::Iter<'_, T>).1: *const T); - _15 = _14 as *const std::ptr::NonNull (PtrToPtr); StorageDead(_14); - _16 = (*_15); - StorageDead(_15); - StorageLive(_18); - StorageLive(_19); - StorageLive(_17); - _17 = ((_13.0: std::slice::Iter<'_, T>).0: std::ptr::NonNull); - _18 = (_17.0: *const T); - StorageDead(_17); - _19 = (_16.0: *const T); - _20 = Eq(_18, _19); - StorageDead(_19); - StorageDead(_18); - goto -> bb7; + _16 = discriminant(_15); + switchInt(move _16) -> [0: bb6, 1: bb8, otherwise: bb10]; } bb6: { - _21 = ((_13.0: std::slice::Iter<'_, T>).1: *const T); - _22 = _21 as usize (Transmute); - _20 = Eq(_22, const 0_usize); - goto -> bb7; + StorageDead(_15); + StorageDead(_13); + drop(_2) -> [return: bb7, unwind unreachable]; } bb7: { - switchInt(move _20) -> [0: bb8, otherwise: bb16]; + return; } bb8: { - StorageLive(_35); - StorageLive(_29); - StorageLive(_31); - StorageLive(_24); - switchInt(const ::IS_ZST) -> [0: bb9, otherwise: bb13]; + _17 = ((_15 as Some).0: &T); + StorageLive(_18); + _18 = &_2; + StorageLive(_19); + _19 = (_17,); + _20 = >::call(move _18, move _19) -> [return: bb9, unwind unreachable]; } bb9: { - StorageLive(_23); - _23 = &raw mut ((_13.0: std::slice::Iter<'_, T>).1: *const T); - _24 = _23 as *mut std::ptr::NonNull (PtrToPtr); - StorageDead(_23); - StorageLive(_28); - _25 = (*_24); - switchInt(const ::IS_ZST) -> [0: bb10, otherwise: bb11]; - } - - bb10: { - StorageLive(_27); - StorageLive(_26); - _26 = (_25.0: *const T); - _27 = Offset(move _26, const -1_isize); - StorageDead(_26); - _28 = NonNull:: { pointer: move _27 }; - StorageDead(_27); - goto -> bb12; - } - - bb11: { - _28 = _25; - goto -> bb12; - } - - bb12: { - (*_24) = move _28; - StorageDead(_28); - _29 = (*_24); - goto -> bb14; - } - - bb13: { - StorageLive(_30); - _30 = &raw mut ((_13.0: std::slice::Iter<'_, T>).1: *const T); - _31 = _30 as *mut usize (PtrToPtr); - StorageDead(_30); - StorageLive(_33); - StorageLive(_32); - _32 = (*_31); - _33 = SubUnchecked(move _32, const 1_usize); - StorageDead(_32); - (*_31) = move _33; - StorageDead(_33); - _29 = ((_13.0: std::slice::Iter<'_, T>).0: std::ptr::NonNull); - goto -> bb14; - } - - bb14: { - StorageDead(_24); - StorageDead(_31); - StorageLive(_34); - _34 = _29; - _35 = (_34.0: *const T); - StorageDead(_34); - _36 = &(*_35); - StorageDead(_29); - StorageDead(_35); - _37 = Option::<&T>::Some(_36); - StorageDead(_20); - StorageDead(_36); - StorageDead(_16); - StorageDead(_21); - StorageDead(_22); - _38 = ((_37 as Some).0: &T); - StorageLive(_39); - _39 = &_2; - StorageLive(_40); - _40 = (_38,); - _41 = >::call(move _39, move _40) -> [return: bb15, unwind unreachable]; - } - - bb15: { - StorageDead(_40); - StorageDead(_39); - StorageDead(_37); + StorageDead(_19); + StorageDead(_18); + StorageDead(_15); goto -> bb4; } - bb16: { - StorageDead(_20); - StorageDead(_36); - StorageDead(_16); - StorageDead(_21); - StorageDead(_22); - StorageDead(_37); - StorageDead(_13); - drop(_2) -> [return: bb17, unwind unreachable]; - } - - bb17: { - return; + bb10: { + unreachable; } } diff --git a/tests/mir-opt/pre-codegen/slice_iter.reverse_loop.PreCodegen.after.panic-unwind.mir b/tests/mir-opt/pre-codegen/slice_iter.reverse_loop.PreCodegen.after.panic-unwind.mir index 7c107a23f9e0b..db9409f72ab1a 100644 --- a/tests/mir-opt/pre-codegen/slice_iter.reverse_loop.PreCodegen.after.panic-unwind.mir +++ b/tests/mir-opt/pre-codegen/slice_iter.reverse_loop.PreCodegen.after.panic-unwind.mir @@ -7,90 +7,19 @@ fn reverse_loop(_1: &[T], _2: impl Fn(&T)) -> () { let mut _11: std::slice::Iter<'_, T>; let mut _12: std::iter::Rev>; let mut _13: std::iter::Rev>; - let mut _37: std::option::Option<&T>; - let mut _39: &impl Fn(&T); - let mut _40: (&T,); - let _41: (); + let mut _15: std::option::Option<&T>; + let mut _16: isize; + let mut _18: &impl Fn(&T); + let mut _19: (&T,); + let _20: (); scope 1 { debug iter => _13; - let _38: &T; + let _17: &T; scope 2 { - debug x => _38; + debug x => _17; } scope 17 (inlined > as Iterator>::next) { - scope 18 (inlined as DoubleEndedIterator>::next_back) { - let mut _14: *const *const T; - let mut _15: *const std::ptr::NonNull; - let mut _20: bool; - let mut _21: *const T; - let _36: &T; - scope 19 { - let _16: std::ptr::NonNull; - let _22: usize; - scope 20 { - } - scope 21 { - scope 25 (inlined as PartialEq>::eq) { - let mut _17: std::ptr::NonNull; - scope 26 (inlined NonNull::::as_ptr) { - let mut _18: *const T; - } - scope 27 (inlined NonNull::::as_ptr) { - let mut _19: *const T; - } - } - } - scope 22 (inlined std::ptr::const_ptr::::addr) { - scope 23 (inlined std::ptr::const_ptr::::cast::<()>) { - } - } - scope 24 (inlined std::ptr::const_ptr::::cast::>) { - } - } - scope 28 (inlined std::slice::Iter::<'_, T>::next_back_unchecked) { - let _29: std::ptr::NonNull; - scope 29 (inlined std::slice::Iter::<'_, T>::pre_dec_end) { - let mut _23: *mut *const T; - let mut _24: *mut std::ptr::NonNull; - let mut _25: std::ptr::NonNull; - let mut _28: std::ptr::NonNull; - let mut _30: *mut *const T; - let mut _31: *mut usize; - let mut _32: usize; - let mut _33: usize; - scope 30 { - scope 31 { - } - scope 32 { - scope 35 (inlined NonNull::::sub) { - scope 36 (inlined core::num::::unchecked_neg) { - scope 37 (inlined core::ub_checks::check_language_ub) { - scope 38 (inlined core::ub_checks::check_language_ub::runtime) { - } - } - } - scope 39 (inlined NonNull::::offset) { - let mut _26: *const T; - let mut _27: *const T; - } - } - } - scope 33 (inlined std::ptr::mut_ptr::::cast::) { - } - scope 34 (inlined std::ptr::mut_ptr::::cast::>) { - } - } - } - scope 40 (inlined NonNull::::as_ref::<'_>) { - let mut _34: std::ptr::NonNull; - scope 41 (inlined NonNull::::as_ptr) { - let mut _35: *const T; - } - scope 42 (inlined std::ptr::mut_ptr::::cast_const) { - } - } - } - } + let mut _14: &mut std::slice::Iter<'_, T>; } } scope 3 (inlined core::slice::::iter) { @@ -178,155 +107,53 @@ fn reverse_loop(_1: &[T], _2: impl Fn(&T)) -> () { } bb4: { - StorageLive(_37); - StorageLive(_22); - StorageLive(_21); - StorageLive(_16); - StorageLive(_36); - StorageLive(_20); - switchInt(const ::IS_ZST) -> [0: bb5, otherwise: bb6]; + StorageLive(_15); + StorageLive(_14); + _14 = &mut (_13.0: std::slice::Iter<'_, T>); + _15 = as DoubleEndedIterator>::next_back(move _14) -> [return: bb5, unwind: bb11]; } bb5: { - StorageLive(_15); - StorageLive(_14); - _14 = &raw const ((_13.0: std::slice::Iter<'_, T>).1: *const T); - _15 = _14 as *const std::ptr::NonNull (PtrToPtr); StorageDead(_14); - _16 = (*_15); - StorageDead(_15); - StorageLive(_18); - StorageLive(_19); - StorageLive(_17); - _17 = ((_13.0: std::slice::Iter<'_, T>).0: std::ptr::NonNull); - _18 = (_17.0: *const T); - StorageDead(_17); - _19 = (_16.0: *const T); - _20 = Eq(_18, _19); - StorageDead(_19); - StorageDead(_18); - goto -> bb7; + _16 = discriminant(_15); + switchInt(move _16) -> [0: bb6, 1: bb8, otherwise: bb10]; } bb6: { - _21 = ((_13.0: std::slice::Iter<'_, T>).1: *const T); - _22 = _21 as usize (Transmute); - _20 = Eq(_22, const 0_usize); - goto -> bb7; + StorageDead(_15); + StorageDead(_13); + drop(_2) -> [return: bb7, unwind continue]; } bb7: { - switchInt(move _20) -> [0: bb8, otherwise: bb18]; + return; } bb8: { - StorageLive(_35); - StorageLive(_29); - StorageLive(_31); - StorageLive(_24); - switchInt(const ::IS_ZST) -> [0: bb9, otherwise: bb13]; + _17 = ((_15 as Some).0: &T); + StorageLive(_18); + _18 = &_2; + StorageLive(_19); + _19 = (_17,); + _20 = >::call(move _18, move _19) -> [return: bb9, unwind: bb11]; } bb9: { - StorageLive(_23); - _23 = &raw mut ((_13.0: std::slice::Iter<'_, T>).1: *const T); - _24 = _23 as *mut std::ptr::NonNull (PtrToPtr); - StorageDead(_23); - StorageLive(_28); - _25 = (*_24); - switchInt(const ::IS_ZST) -> [0: bb10, otherwise: bb11]; + StorageDead(_19); + StorageDead(_18); + StorageDead(_15); + goto -> bb4; } bb10: { - StorageLive(_27); - StorageLive(_26); - _26 = (_25.0: *const T); - _27 = Offset(move _26, const -1_isize); - StorageDead(_26); - _28 = NonNull:: { pointer: move _27 }; - StorageDead(_27); - goto -> bb12; - } - - bb11: { - _28 = _25; - goto -> bb12; - } - - bb12: { - (*_24) = move _28; - StorageDead(_28); - _29 = (*_24); - goto -> bb14; - } - - bb13: { - StorageLive(_30); - _30 = &raw mut ((_13.0: std::slice::Iter<'_, T>).1: *const T); - _31 = _30 as *mut usize (PtrToPtr); - StorageDead(_30); - StorageLive(_33); - StorageLive(_32); - _32 = (*_31); - _33 = SubUnchecked(move _32, const 1_usize); - StorageDead(_32); - (*_31) = move _33; - StorageDead(_33); - _29 = ((_13.0: std::slice::Iter<'_, T>).0: std::ptr::NonNull); - goto -> bb14; + unreachable; } - bb14: { - StorageDead(_24); - StorageDead(_31); - StorageLive(_34); - _34 = _29; - _35 = (_34.0: *const T); - StorageDead(_34); - _36 = &(*_35); - StorageDead(_29); - StorageDead(_35); - _37 = Option::<&T>::Some(_36); - StorageDead(_20); - StorageDead(_36); - StorageDead(_16); - StorageDead(_21); - StorageDead(_22); - _38 = ((_37 as Some).0: &T); - StorageLive(_39); - _39 = &_2; - StorageLive(_40); - _40 = (_38,); - _41 = >::call(move _39, move _40) -> [return: bb15, unwind: bb16]; + bb11 (cleanup): { + drop(_2) -> [return: bb12, unwind terminate(cleanup)]; } - bb15: { - StorageDead(_40); - StorageDead(_39); - StorageDead(_37); - goto -> bb4; - } - - bb16 (cleanup): { - drop(_2) -> [return: bb17, unwind terminate(cleanup)]; - } - - bb17 (cleanup): { + bb12 (cleanup): { resume; } - - bb18: { - StorageDead(_20); - StorageDead(_36); - StorageDead(_16); - StorageDead(_21); - StorageDead(_22); - StorageDead(_37); - StorageDead(_13); - drop(_2) -> [return: bb19, unwind continue]; - } - - bb19: { - return; - } } diff --git a/tests/mir-opt/pre-codegen/slice_iter.slice_iter_mut_next_back.PreCodegen.after.panic-abort.mir b/tests/mir-opt/pre-codegen/slice_iter.slice_iter_mut_next_back.PreCodegen.after.panic-abort.mir index 2df346c081c8e..78f96bf419559 100644 --- a/tests/mir-opt/pre-codegen/slice_iter.slice_iter_mut_next_back.PreCodegen.after.panic-abort.mir +++ b/tests/mir-opt/pre-codegen/slice_iter.slice_iter_mut_next_back.PreCodegen.after.panic-abort.mir @@ -3,205 +3,12 @@ fn slice_iter_mut_next_back(_1: &mut std::slice::IterMut<'_, T>) -> Option<&mut T> { debug it => _1; let mut _0: std::option::Option<&mut T>; - scope 1 (inlined as DoubleEndedIterator>::next_back) { - let mut _2: *const *mut T; - let mut _3: *const std::ptr::NonNull; - let mut _8: bool; - let mut _9: *mut T; - let mut _25: &mut T; - scope 2 { - let _4: std::ptr::NonNull; - let _10: usize; - scope 3 { - } - scope 4 { - scope 8 (inlined as PartialEq>::eq) { - let mut _5: std::ptr::NonNull; - scope 9 (inlined NonNull::::as_ptr) { - let mut _6: *const T; - } - scope 10 (inlined NonNull::::as_ptr) { - let mut _7: *const T; - } - } - } - scope 5 (inlined std::ptr::mut_ptr::::addr) { - scope 6 (inlined std::ptr::mut_ptr::::cast::<()>) { - } - } - scope 7 (inlined std::ptr::const_ptr::::cast::>) { - } - } - scope 11 (inlined std::slice::IterMut::<'_, T>::next_back_unchecked) { - let mut _17: std::ptr::NonNull; - scope 12 (inlined std::slice::IterMut::<'_, T>::pre_dec_end) { - let mut _11: *mut *mut T; - let mut _12: *mut std::ptr::NonNull; - let mut _13: std::ptr::NonNull; - let mut _16: std::ptr::NonNull; - let mut _18: *mut *mut T; - let mut _19: *mut usize; - let mut _20: usize; - let mut _21: usize; - scope 13 { - scope 14 { - } - scope 15 { - scope 18 (inlined NonNull::::sub) { - scope 19 (inlined core::num::::unchecked_neg) { - scope 20 (inlined core::ub_checks::check_language_ub) { - scope 21 (inlined core::ub_checks::check_language_ub::runtime) { - } - } - } - scope 22 (inlined NonNull::::offset) { - let mut _14: *const T; - let mut _15: *const T; - } - } - } - scope 16 (inlined std::ptr::mut_ptr::::cast::) { - } - scope 17 (inlined std::ptr::mut_ptr::::cast::>) { - } - } - } - scope 23 (inlined NonNull::::as_mut::<'_>) { - let mut _22: std::ptr::NonNull; - let mut _24: *mut T; - scope 24 (inlined NonNull::::as_ptr) { - let mut _23: *const T; - } - } - } - } bb0: { - StorageLive(_10); - StorageLive(_9); - StorageLive(_4); - StorageLive(_25); - StorageLive(_8); - switchInt(const ::IS_ZST) -> [0: bb1, otherwise: bb2]; + _0 = as DoubleEndedIterator>::next_back(move _1) -> [return: bb1, unwind unreachable]; } bb1: { - StorageLive(_3); - StorageLive(_2); - _2 = &raw const ((*_1).1: *mut T); - _3 = _2 as *const std::ptr::NonNull (PtrToPtr); - StorageDead(_2); - _4 = (*_3); - StorageDead(_3); - StorageLive(_6); - StorageLive(_7); - StorageLive(_5); - _5 = ((*_1).0: std::ptr::NonNull); - _6 = (_5.0: *const T); - StorageDead(_5); - _7 = (_4.0: *const T); - _8 = Eq(_6, _7); - StorageDead(_7); - StorageDead(_6); - goto -> bb3; - } - - bb2: { - _9 = ((*_1).1: *mut T); - _10 = _9 as usize (Transmute); - _8 = Eq(_10, const 0_usize); - goto -> bb3; - } - - bb3: { - switchInt(move _8) -> [0: bb4, otherwise: bb11]; - } - - bb4: { - StorageLive(_24); - StorageLive(_17); - StorageLive(_19); - StorageLive(_12); - switchInt(const ::IS_ZST) -> [0: bb5, otherwise: bb9]; - } - - bb5: { - StorageLive(_11); - _11 = &raw mut ((*_1).1: *mut T); - _12 = _11 as *mut std::ptr::NonNull (PtrToPtr); - StorageDead(_11); - StorageLive(_16); - _13 = (*_12); - switchInt(const ::IS_ZST) -> [0: bb6, otherwise: bb7]; - } - - bb6: { - StorageLive(_15); - StorageLive(_14); - _14 = (_13.0: *const T); - _15 = Offset(move _14, const -1_isize); - StorageDead(_14); - _16 = NonNull:: { pointer: move _15 }; - StorageDead(_15); - goto -> bb8; - } - - bb7: { - _16 = _13; - goto -> bb8; - } - - bb8: { - (*_12) = move _16; - StorageDead(_16); - _17 = (*_12); - goto -> bb10; - } - - bb9: { - StorageLive(_18); - _18 = &raw mut ((*_1).1: *mut T); - _19 = _18 as *mut usize (PtrToPtr); - StorageDead(_18); - StorageLive(_21); - StorageLive(_20); - _20 = (*_19); - _21 = SubUnchecked(move _20, const 1_usize); - StorageDead(_20); - (*_19) = move _21; - StorageDead(_21); - _17 = ((*_1).0: std::ptr::NonNull); - goto -> bb10; - } - - bb10: { - StorageDead(_12); - StorageDead(_19); - StorageLive(_22); - _22 = _17; - StorageLive(_23); - _23 = (_22.0: *const T); - _24 = move _23 as *mut T (PtrToPtr); - StorageDead(_23); - StorageDead(_22); - _25 = &mut (*_24); - StorageDead(_17); - StorageDead(_24); - _0 = Option::<&mut T>::Some(_25); - goto -> bb12; - } - - bb11: { - _0 = const {transmute(0x0000000000000000): Option<&mut T>}; - goto -> bb12; - } - - bb12: { - StorageDead(_8); - StorageDead(_25); - StorageDead(_4); - StorageDead(_9); - StorageDead(_10); return; } } diff --git a/tests/mir-opt/pre-codegen/slice_iter.slice_iter_mut_next_back.PreCodegen.after.panic-unwind.mir b/tests/mir-opt/pre-codegen/slice_iter.slice_iter_mut_next_back.PreCodegen.after.panic-unwind.mir index 2df346c081c8e..dfe5e206fadaf 100644 --- a/tests/mir-opt/pre-codegen/slice_iter.slice_iter_mut_next_back.PreCodegen.after.panic-unwind.mir +++ b/tests/mir-opt/pre-codegen/slice_iter.slice_iter_mut_next_back.PreCodegen.after.panic-unwind.mir @@ -3,205 +3,12 @@ fn slice_iter_mut_next_back(_1: &mut std::slice::IterMut<'_, T>) -> Option<&mut T> { debug it => _1; let mut _0: std::option::Option<&mut T>; - scope 1 (inlined as DoubleEndedIterator>::next_back) { - let mut _2: *const *mut T; - let mut _3: *const std::ptr::NonNull; - let mut _8: bool; - let mut _9: *mut T; - let mut _25: &mut T; - scope 2 { - let _4: std::ptr::NonNull; - let _10: usize; - scope 3 { - } - scope 4 { - scope 8 (inlined as PartialEq>::eq) { - let mut _5: std::ptr::NonNull; - scope 9 (inlined NonNull::::as_ptr) { - let mut _6: *const T; - } - scope 10 (inlined NonNull::::as_ptr) { - let mut _7: *const T; - } - } - } - scope 5 (inlined std::ptr::mut_ptr::::addr) { - scope 6 (inlined std::ptr::mut_ptr::::cast::<()>) { - } - } - scope 7 (inlined std::ptr::const_ptr::::cast::>) { - } - } - scope 11 (inlined std::slice::IterMut::<'_, T>::next_back_unchecked) { - let mut _17: std::ptr::NonNull; - scope 12 (inlined std::slice::IterMut::<'_, T>::pre_dec_end) { - let mut _11: *mut *mut T; - let mut _12: *mut std::ptr::NonNull; - let mut _13: std::ptr::NonNull; - let mut _16: std::ptr::NonNull; - let mut _18: *mut *mut T; - let mut _19: *mut usize; - let mut _20: usize; - let mut _21: usize; - scope 13 { - scope 14 { - } - scope 15 { - scope 18 (inlined NonNull::::sub) { - scope 19 (inlined core::num::::unchecked_neg) { - scope 20 (inlined core::ub_checks::check_language_ub) { - scope 21 (inlined core::ub_checks::check_language_ub::runtime) { - } - } - } - scope 22 (inlined NonNull::::offset) { - let mut _14: *const T; - let mut _15: *const T; - } - } - } - scope 16 (inlined std::ptr::mut_ptr::::cast::) { - } - scope 17 (inlined std::ptr::mut_ptr::::cast::>) { - } - } - } - scope 23 (inlined NonNull::::as_mut::<'_>) { - let mut _22: std::ptr::NonNull; - let mut _24: *mut T; - scope 24 (inlined NonNull::::as_ptr) { - let mut _23: *const T; - } - } - } - } bb0: { - StorageLive(_10); - StorageLive(_9); - StorageLive(_4); - StorageLive(_25); - StorageLive(_8); - switchInt(const ::IS_ZST) -> [0: bb1, otherwise: bb2]; + _0 = as DoubleEndedIterator>::next_back(move _1) -> [return: bb1, unwind continue]; } bb1: { - StorageLive(_3); - StorageLive(_2); - _2 = &raw const ((*_1).1: *mut T); - _3 = _2 as *const std::ptr::NonNull (PtrToPtr); - StorageDead(_2); - _4 = (*_3); - StorageDead(_3); - StorageLive(_6); - StorageLive(_7); - StorageLive(_5); - _5 = ((*_1).0: std::ptr::NonNull); - _6 = (_5.0: *const T); - StorageDead(_5); - _7 = (_4.0: *const T); - _8 = Eq(_6, _7); - StorageDead(_7); - StorageDead(_6); - goto -> bb3; - } - - bb2: { - _9 = ((*_1).1: *mut T); - _10 = _9 as usize (Transmute); - _8 = Eq(_10, const 0_usize); - goto -> bb3; - } - - bb3: { - switchInt(move _8) -> [0: bb4, otherwise: bb11]; - } - - bb4: { - StorageLive(_24); - StorageLive(_17); - StorageLive(_19); - StorageLive(_12); - switchInt(const ::IS_ZST) -> [0: bb5, otherwise: bb9]; - } - - bb5: { - StorageLive(_11); - _11 = &raw mut ((*_1).1: *mut T); - _12 = _11 as *mut std::ptr::NonNull (PtrToPtr); - StorageDead(_11); - StorageLive(_16); - _13 = (*_12); - switchInt(const ::IS_ZST) -> [0: bb6, otherwise: bb7]; - } - - bb6: { - StorageLive(_15); - StorageLive(_14); - _14 = (_13.0: *const T); - _15 = Offset(move _14, const -1_isize); - StorageDead(_14); - _16 = NonNull:: { pointer: move _15 }; - StorageDead(_15); - goto -> bb8; - } - - bb7: { - _16 = _13; - goto -> bb8; - } - - bb8: { - (*_12) = move _16; - StorageDead(_16); - _17 = (*_12); - goto -> bb10; - } - - bb9: { - StorageLive(_18); - _18 = &raw mut ((*_1).1: *mut T); - _19 = _18 as *mut usize (PtrToPtr); - StorageDead(_18); - StorageLive(_21); - StorageLive(_20); - _20 = (*_19); - _21 = SubUnchecked(move _20, const 1_usize); - StorageDead(_20); - (*_19) = move _21; - StorageDead(_21); - _17 = ((*_1).0: std::ptr::NonNull); - goto -> bb10; - } - - bb10: { - StorageDead(_12); - StorageDead(_19); - StorageLive(_22); - _22 = _17; - StorageLive(_23); - _23 = (_22.0: *const T); - _24 = move _23 as *mut T (PtrToPtr); - StorageDead(_23); - StorageDead(_22); - _25 = &mut (*_24); - StorageDead(_17); - StorageDead(_24); - _0 = Option::<&mut T>::Some(_25); - goto -> bb12; - } - - bb11: { - _0 = const {transmute(0x0000000000000000): Option<&mut T>}; - goto -> bb12; - } - - bb12: { - StorageDead(_8); - StorageDead(_25); - StorageDead(_4); - StorageDead(_9); - StorageDead(_10); return; } }