11//! Macros used by iterators of slice.
22
3- // Shrinks the iterator when T is a ZST, setting the length to `new_len`.
4- // `new_len` must not exceed `self.len()`.
5- macro_rules! zst_set_len {
6- ( $self: ident, $new_len: expr) => { {
3+ /// Convenience & performance macro for consuming the `end_or_len` field, by
4+ /// giving a `(&mut) usize` or `(&mut) NonNull<T>` depending whether `T` is
5+ /// or is not a ZST respectively.
6+ ///
7+ /// Internally, this reads the `end` through a pointer-to-`NonNull` so that
8+ /// it'll get the appropriate non-null metadata in the backend without needing
9+ /// to call `assume` manually.
10+ macro_rules! if_zst {
11+ ( mut $this: ident, $len: ident => $zst_body: expr, $end: ident => $other_body: expr, ) => { {
712 #![ allow( unused_unsafe) ] // we're sometimes used within an unsafe block
813
9- // SAFETY: same as `invalid(_mut)`, but the macro doesn't know
10- // which versions of that function to call, so open-code it.
11- $self. end = unsafe { mem:: transmute:: <usize , _>( $new_len) } ;
14+ if T :: IS_ZST {
15+ // SAFETY: for ZSTs, the pointer is storing a provenance-free length,
16+ // so consuming and updating it as a `usize` is fine.
17+ let $len = unsafe { & mut * ptr:: addr_of_mut!( $this. end_or_len) . cast:: <usize >( ) } ;
18+ $zst_body
19+ } else {
20+ // SAFETY: for non-ZSTs, the type invariant ensures it cannot be null
21+ let $end = unsafe { & mut * ptr:: addr_of_mut!( $this. end_or_len) . cast:: <NonNull <T >>( ) } ;
22+ $other_body
23+ }
1224 } } ;
13- }
25+ ( $this: ident, $len: ident => $zst_body: expr, $end: ident => $other_body: expr, ) => { {
26+ #![ allow( unused_unsafe) ] // we're sometimes used within an unsafe block
1427
15- // Shrinks the iterator when T is a ZST, reducing the length by `n`.
16- // `n` must not exceed `self.len()`.
17- macro_rules! zst_shrink {
18- ( $self: ident, $n: ident) => {
19- let new_len = $self. end. addr( ) - $n;
20- zst_set_len!( $self, new_len) ;
21- } ;
28+ if T :: IS_ZST {
29+ let $len = $this. end_or_len. addr( ) ;
30+ $zst_body
31+ } else {
32+ // SAFETY: for non-ZSTs, the type invariant ensures it cannot be null
33+ let $end = unsafe { * ptr:: addr_of!( $this. end_or_len) . cast:: <NonNull <T >>( ) } ;
34+ $other_body
35+ }
36+ } } ;
2237}
2338
2439// Inlining is_empty and len makes a huge performance difference
2540macro_rules! is_empty {
2641 ( $self: ident) => {
27- if T :: IS_ZST { $self. end. addr( ) == 0 } else { $self. ptr. as_ptr( ) as * const _ == $self. end }
42+ if_zst!( $self,
43+ len => len == 0 ,
44+ end => $self. ptr == end,
45+ )
2846 } ;
2947}
3048
3149macro_rules! len {
3250 ( $self: ident) => { {
33- #![ allow( unused_unsafe) ] // we're sometimes used within an unsafe block
34-
35- if T :: IS_ZST {
36- $self. end. addr( )
37- } else {
38- // To get rid of some bounds checks (see `position`), we use ptr_sub instead of
39- // offset_from (Tested by `codegen/slice-position-bounds-check`.)
40- // SAFETY: by the type invariant pointers are aligned and `start <= end`
41- unsafe { $self. end. sub_ptr( $self. ptr. as_ptr( ) ) }
42- }
51+ if_zst!( $self,
52+ len => len,
53+ end => {
54+ // To get rid of some bounds checks (see `position`), we use ptr_sub instead of
55+ // offset_from (Tested by `codegen/slice-position-bounds-check`.)
56+ // SAFETY: by the type invariant pointers are aligned and `start <= end`
57+ unsafe { end. sub_ptr( $self. ptr) }
58+ } ,
59+ )
4360 } } ;
4461}
4562
@@ -50,20 +67,21 @@ macro_rules! iterator {
5067 $elem: ty,
5168 $raw_mut: tt,
5269 { $( $mut_: tt ) ?} ,
70+ $into_ref: ident,
5371 { $( $extra: tt) * }
5472 ) => {
5573 // Returns the first element and moves the start of the iterator forwards by 1.
5674 // Greatly improves performance compared to an inlined function. The iterator
5775 // must not be empty.
5876 macro_rules! next_unchecked {
59- ( $self: ident) => { & $ ( $mut_ ) ? * $ self. post_inc_start( 1 ) }
77+ ( $self: ident) => { $ self. post_inc_start( 1 ) . $into_ref ( ) }
6078 }
6179
6280 // Returns the last element and moves the end of the iterator backwards by 1.
6381 // Greatly improves performance compared to an inlined function. The iterator
6482 // must not be empty.
6583 macro_rules! next_back_unchecked {
66- ( $self: ident) => { & $ ( $mut_ ) ? * $ self. pre_dec_end( 1 ) }
84+ ( $self: ident) => { $ self. pre_dec_end( 1 ) . $into_ref ( ) }
6785 }
6886
6987 impl <' a, T > $name<' a, T > {
@@ -80,33 +98,40 @@ macro_rules! iterator {
8098 // returning the old start.
8199 // Unsafe because the offset must not exceed `self.len()`.
82100 #[ inline( always) ]
83- unsafe fn post_inc_start( & mut self , offset: usize ) -> * $raw_mut T {
101+ unsafe fn post_inc_start( & mut self , offset: usize ) -> NonNull < T > {
84102 let old = self . ptr;
85- if T :: IS_ZST {
86- zst_shrink!( self , offset) ;
87- } else {
88- // SAFETY: the caller guarantees that `offset` doesn't exceed `self.len()`,
89- // so this new pointer is inside `self` and thus guaranteed to be non-null.
90- self . ptr = unsafe { self . ptr. add( offset) } ;
103+
104+ // SAFETY: the caller guarantees that `offset` doesn't exceed `self.len()`,
105+ // so this new pointer is inside `self` and thus guaranteed to be non-null.
106+ unsafe {
107+ if_zst!( mut self ,
108+ len => * len = len. unchecked_sub( offset) ,
109+ _end => self . ptr = self . ptr. add( offset) ,
110+ ) ;
91111 }
92- old. as_ptr ( )
112+ old
93113 }
94114
95115 // Helper function for moving the end of the iterator backwards by `offset` elements,
96116 // returning the new end.
97117 // Unsafe because the offset must not exceed `self.len()`.
98118 #[ inline( always) ]
99- unsafe fn pre_dec_end( & mut self , offset: usize ) -> * $raw_mut T {
100- if T :: IS_ZST {
101- zst_shrink!( self , offset) ;
102- self . ptr. as_ptr( )
103- } else {
119+ unsafe fn pre_dec_end( & mut self , offset: usize ) -> NonNull <T > {
120+ if_zst!( mut self ,
121+ // SAFETY: By our precondition, `offset` can be at most the
122+ // current length, so the subtraction can never overflow.
123+ len => unsafe {
124+ * len = len. unchecked_sub( offset) ;
125+ self . ptr
126+ } ,
104127 // SAFETY: the caller guarantees that `offset` doesn't exceed `self.len()`,
105128 // which is guaranteed to not overflow an `isize`. Also, the resulting pointer
106129 // is in bounds of `slice`, which fulfills the other requirements for `offset`.
107- self . end = unsafe { self . end. sub( offset) } ;
108- self . end
109- }
130+ end => unsafe {
131+ * end = end. sub( offset) ;
132+ * end
133+ } ,
134+ )
110135 }
111136 }
112137
@@ -131,13 +156,9 @@ macro_rules! iterator {
131156 fn next( & mut self ) -> Option <$elem> {
132157 // could be implemented with slices, but this avoids bounds checks
133158
134- // SAFETY: `assume` call is safe because slices over non-ZSTs must
135- // have a non-null end pointer. The call to `next_unchecked!` is
159+ // SAFETY: The call to `next_unchecked!` is
136160 // safe since we check if the iterator is empty first.
137161 unsafe {
138- if !<T >:: IS_ZST {
139- assume( !self . end. is_null( ) ) ;
140- }
141162 if is_empty!( self ) {
142163 None
143164 } else {
@@ -161,14 +182,10 @@ macro_rules! iterator {
161182 fn nth( & mut self , n: usize ) -> Option <$elem> {
162183 if n >= len!( self ) {
163184 // This iterator is now empty.
164- if T :: IS_ZST {
165- zst_set_len!( self , 0 ) ;
166- } else {
167- // SAFETY: end can't be 0 if T isn't ZST because ptr isn't 0 and end >= ptr
168- unsafe {
169- self . ptr = NonNull :: new_unchecked( self . end as * mut T ) ;
170- }
171- }
185+ if_zst!( mut self ,
186+ len => * len = 0 ,
187+ end => self . ptr = * end,
188+ ) ;
172189 return None ;
173190 }
174191 // SAFETY: We are in bounds. `post_inc_start` does the right thing even for ZSTs.
@@ -375,13 +392,9 @@ macro_rules! iterator {
375392 fn next_back( & mut self ) -> Option <$elem> {
376393 // could be implemented with slices, but this avoids bounds checks
377394
378- // SAFETY: `assume` call is safe because slices over non-ZSTs must
379- // have a non-null end pointer. The call to `next_back_unchecked!`
395+ // SAFETY: The call to `next_back_unchecked!`
380396 // is safe since we check if the iterator is empty first.
381397 unsafe {
382- if !<T >:: IS_ZST {
383- assume( !self . end. is_null( ) ) ;
384- }
385398 if is_empty!( self ) {
386399 None
387400 } else {
@@ -394,11 +407,10 @@ macro_rules! iterator {
394407 fn nth_back( & mut self , n: usize ) -> Option <$elem> {
395408 if n >= len!( self ) {
396409 // This iterator is now empty.
397- if T :: IS_ZST {
398- zst_set_len!( self , 0 ) ;
399- } else {
400- self . end = self . ptr. as_ptr( ) ;
401- }
410+ if_zst!( mut self ,
411+ len => * len = 0 ,
412+ end => * end = self . ptr,
413+ ) ;
402414 return None ;
403415 }
404416 // SAFETY: We are in bounds. `pre_dec_end` does the right thing even for ZSTs.
0 commit comments