@@ -133,21 +133,20 @@ const QUEUE_LOCKED: usize = 4;
133133const SINGLE : usize = 8 ;
134134const MASK : usize = !( QUEUE_LOCKED | QUEUED | LOCKED ) ;
135135
136- /// Returns a closure that changes the state to the lock state corresponding to
137- /// the lock mode indicated in `write`.
136+ /// Marks the state as write-locked, if possible.
138137#[ inline]
139- fn lock ( write : bool ) -> impl Fn ( State ) -> Option < State > {
140- move | state| {
141- if write {
142- let state = state . wrapping_byte_add ( LOCKED ) ;
143- if state . addr ( ) & LOCKED == LOCKED { Some ( state ) } else { None }
144- } else {
145- if state . addr ( ) & QUEUED == 0 && state . addr ( ) != LOCKED {
146- Some ( invalid_mut ( state. addr ( ) . checked_add ( SINGLE ) ? | LOCKED ) )
147- } else {
148- None
149- }
150- }
138+ fn write_lock ( state : State ) -> Option < State > {
139+ let state = state . wrapping_byte_add ( LOCKED ) ;
140+ if state . addr ( ) & LOCKED == LOCKED { Some ( state ) } else { None }
141+ }
142+
143+ /// Marks the state as read-locked, if possible.
144+ # [ inline ]
145+ fn read_lock ( state : State ) -> Option < State > {
146+ if state . addr ( ) & QUEUED == 0 && state . addr ( ) != LOCKED {
147+ Some ( invalid_mut ( state . addr ( ) . checked_add ( SINGLE ) ? | LOCKED ) )
148+ } else {
149+ None
151150 }
152151}
153152
@@ -241,7 +240,7 @@ impl Drop for PanicGuard {
241240 }
242241}
243242
244- /// Find the tail of the queue beginning with `head`, caching the result in `head` .
243+ /// Add backlinks to the queue, returning the tail .
245244///
246245/// May be called from multiple threads at the same time, while the queue is not
247246/// modified (this happens when unlocking multiple readers).
@@ -252,7 +251,7 @@ impl Drop for PanicGuard {
252251/// last removal.
253252/// * The part of the queue starting with `head` must not be modified during this
254253/// call.
255- unsafe fn find_tail ( head : NonNull < Node > ) -> NonNull < Node > {
254+ unsafe fn add_backlinks_and_find_tail ( head : NonNull < Node > ) -> NonNull < Node > {
256255 let mut current = head;
257256 let tail = loop {
258257 let c = unsafe { current. as_ref ( ) } ;
@@ -287,7 +286,7 @@ impl RwLock {
287286
288287 #[ inline]
289288 pub fn try_read ( & self ) -> bool {
290- self . state . fetch_update ( Acquire , Relaxed , lock ( false ) ) . is_ok ( )
289+ self . state . fetch_update ( Acquire , Relaxed , read_lock ) . is_ok ( )
291290 }
292291
293292 #[ inline]
@@ -316,7 +315,7 @@ impl RwLock {
316315
317316 #[ cold]
318317 fn lock_contended ( & self , write : bool ) {
319- let update = lock ( write) ;
318+ let update = if write { write_lock } else { read_lock } ;
320319 let mut node = Node :: new ( write) ;
321320 let mut state = self . state . load ( Relaxed ) ;
322321 let mut count = 0 ;
@@ -428,7 +427,7 @@ impl RwLock {
428427 // all queue-lock owners will observe the set `LOCKED` bit. Because they
429428 // do not modify the queue while there is a lock owner, the queue will
430429 // not be removed from here.
431- let tail = unsafe { find_tail ( to_node ( state) ) . as_ref ( ) } ;
430+ let tail = unsafe { add_backlinks_and_find_tail ( to_node ( state) ) . as_ref ( ) } ;
432431 // The lock count is stored in the `next` field of `tail`.
433432 // Decrement it, making sure to observe all changes made to the queue
434433 // by the other lock owners by using acquire-release ordering.
@@ -482,8 +481,7 @@ impl RwLock {
482481 debug_assert_eq ! ( state. addr( ) & ( QUEUED | QUEUE_LOCKED ) , QUEUED | QUEUE_LOCKED ) ;
483482
484483 loop {
485- // Find the last node in the linked list.
486- let tail = unsafe { find_tail ( to_node ( state) ) } ;
484+ let tail = unsafe { add_backlinks_and_find_tail ( to_node ( state) ) } ;
487485
488486 if state. addr ( ) & LOCKED == LOCKED {
489487 // Another thread has locked the lock. Leave waking up waiters
0 commit comments