@@ -3014,6 +3014,28 @@ macro_rules! iterator {
30143014 { $( $mut_: tt ) * } ,
30153015 { $( $extra: tt) * }
30163016 ) => {
3017+ // Returns the first element and moves the start of the iterator forwards by 1.
3018+ // Greatly improves performance compared to an inlined function. The iterator
3019+ // must not be empty.
3020+ macro_rules! next_unchecked {
3021+ ( $self: ident) => { & $( $mut_ ) * * $self. post_inc_start( 1 ) }
3022+ }
3023+
3024+ // Returns the last element and moves the end of the iterator backwards by 1.
3025+ // Greatly improves performance compared to an inlined function. The iterator
3026+ // must not be empty.
3027+ macro_rules! next_back_unchecked {
3028+ ( $self: ident) => { & $( $mut_ ) * * $self. pre_dec_end( 1 ) }
3029+ }
3030+
3031+ // Shrinks the iterator when T is a ZST, by moving the end of the iterator
3032+ // backwards by `n`. `n` must not exceed `self.len()`.
3033+ macro_rules! zst_shrink {
3034+ ( $self: ident, $n: ident) => {
3035+ $self. end = ( $self. end as * $raw_mut u8 ) . wrapping_offset( -$n) as * $raw_mut T ;
3036+ }
3037+ }
3038+
30173039 impl <' a, T > $name<' a, T > {
30183040 // Helper function for creating a slice from the iterator.
30193041 #[ inline( always) ]
@@ -3023,12 +3045,11 @@ macro_rules! iterator {
30233045
30243046 // Helper function for moving the start of the iterator forwards by `offset` elements,
30253047 // returning the old start.
3026- // Unsafe because the offset must be in-bounds or one-past-the-end .
3048+ // Unsafe because the offset must not exceed `self.len()` .
30273049 #[ inline( always) ]
30283050 unsafe fn post_inc_start( & mut self , offset: isize ) -> * $raw_mut T {
30293051 if mem:: size_of:: <T >( ) == 0 {
3030- // This is *reducing* the length. `ptr` never changes with ZST.
3031- self . end = ( self . end as * $raw_mut u8 ) . wrapping_offset( -offset) as * $raw_mut T ;
3052+ zst_shrink!( self , offset) ;
30323053 self . ptr
30333054 } else {
30343055 let old = self . ptr;
@@ -3039,11 +3060,11 @@ macro_rules! iterator {
30393060
30403061 // Helper function for moving the end of the iterator backwards by `offset` elements,
30413062 // returning the new end.
3042- // Unsafe because the offset must be in-bounds or one-past-the-end .
3063+ // Unsafe because the offset must not exceed `self.len()` .
30433064 #[ inline( always) ]
30443065 unsafe fn pre_dec_end( & mut self , offset: isize ) -> * $raw_mut T {
30453066 if mem:: size_of:: <T >( ) == 0 {
3046- self . end = ( self . end as * $raw_mut u8 ) . wrapping_offset ( - offset) as * $raw_mut T ;
3067+ zst_shrink! ( self , offset) ;
30473068 self . ptr
30483069 } else {
30493070 self . end = self . end. offset( -offset) ;
@@ -3080,7 +3101,7 @@ macro_rules! iterator {
30803101 if is_empty!( self ) {
30813102 None
30823103 } else {
3083- Some ( & $ ( $mut_ ) * * self . post_inc_start ( 1 ) )
3104+ Some ( next_unchecked! ( self ) )
30843105 }
30853106 }
30863107 }
@@ -3109,11 +3130,10 @@ macro_rules! iterator {
31093130 }
31103131 return None ;
31113132 }
3112- // We are in bounds. `offset ` does the right thing even for ZSTs.
3133+ // We are in bounds. `post_inc_start ` does the right thing even for ZSTs.
31133134 unsafe {
3114- let elem = Some ( & $( $mut_ ) * * self . ptr. add( n) ) ;
3115- self . post_inc_start( ( n as isize ) . wrapping_add( 1 ) ) ;
3116- elem
3135+ self . post_inc_start( n as isize ) ;
3136+ Some ( next_unchecked!( self ) )
31173137 }
31183138 }
31193139
@@ -3130,13 +3150,13 @@ macro_rules! iterator {
31303150 let mut accum = init;
31313151 unsafe {
31323152 while len!( self ) >= 4 {
3133- accum = f( accum, & $ ( $mut_ ) * * self . post_inc_start ( 1 ) ) ?;
3134- accum = f( accum, & $ ( $mut_ ) * * self . post_inc_start ( 1 ) ) ?;
3135- accum = f( accum, & $ ( $mut_ ) * * self . post_inc_start ( 1 ) ) ?;
3136- accum = f( accum, & $ ( $mut_ ) * * self . post_inc_start ( 1 ) ) ?;
3153+ accum = f( accum, next_unchecked! ( self ) ) ?;
3154+ accum = f( accum, next_unchecked! ( self ) ) ?;
3155+ accum = f( accum, next_unchecked! ( self ) ) ?;
3156+ accum = f( accum, next_unchecked! ( self ) ) ?;
31373157 }
31383158 while !is_empty!( self ) {
3139- accum = f( accum, & $ ( $mut_ ) * * self . post_inc_start ( 1 ) ) ?;
3159+ accum = f( accum, next_unchecked! ( self ) ) ?;
31403160 }
31413161 }
31423162 Try :: from_ok( accum)
@@ -3207,11 +3227,25 @@ macro_rules! iterator {
32073227 if is_empty!( self ) {
32083228 None
32093229 } else {
3210- Some ( & $ ( $mut_ ) * * self . pre_dec_end ( 1 ) )
3230+ Some ( next_back_unchecked! ( self ) )
32113231 }
32123232 }
32133233 }
32143234
3235+ #[ inline]
3236+ fn nth_back( & mut self , n: usize ) -> Option <$elem> {
3237+ if n >= len!( self ) {
3238+ // This iterator is now empty.
3239+ self . end = self . ptr;
3240+ return None ;
3241+ }
3242+ // We are in bounds. `pre_dec_end` does the right thing even for ZSTs.
3243+ unsafe {
3244+ self . pre_dec_end( n as isize ) ;
3245+ Some ( next_back_unchecked!( self ) )
3246+ }
3247+ }
3248+
32153249 #[ inline]
32163250 fn try_rfold<B , F , R >( & mut self , init: B , mut f: F ) -> R where
32173251 Self : Sized , F : FnMut ( B , Self :: Item ) -> R , R : Try <Ok =B >
@@ -3220,14 +3254,14 @@ macro_rules! iterator {
32203254 let mut accum = init;
32213255 unsafe {
32223256 while len!( self ) >= 4 {
3223- accum = f( accum, & $ ( $mut_ ) * * self . pre_dec_end ( 1 ) ) ?;
3224- accum = f( accum, & $ ( $mut_ ) * * self . pre_dec_end ( 1 ) ) ?;
3225- accum = f( accum, & $ ( $mut_ ) * * self . pre_dec_end ( 1 ) ) ?;
3226- accum = f( accum, & $ ( $mut_ ) * * self . pre_dec_end ( 1 ) ) ?;
3257+ accum = f( accum, next_back_unchecked! ( self ) ) ?;
3258+ accum = f( accum, next_back_unchecked! ( self ) ) ?;
3259+ accum = f( accum, next_back_unchecked! ( self ) ) ?;
3260+ accum = f( accum, next_back_unchecked! ( self ) ) ?;
32273261 }
32283262 // inlining is_empty everywhere makes a huge performance difference
32293263 while !is_empty!( self ) {
3230- accum = f( accum, & $ ( $mut_ ) * * self . pre_dec_end ( 1 ) ) ?;
3264+ accum = f( accum, next_back_unchecked! ( self ) ) ?;
32313265 }
32323266 }
32333267 Try :: from_ok( accum)
0 commit comments