@@ -354,7 +354,7 @@ impl Once {
354354 // SeqCst minimizes the chances of something going wrong.
355355 let mut state_and_queue = self . state_and_queue . load ( Ordering :: SeqCst ) ;
356356
357- ' outer : loop {
357+ loop {
358358 match state_and_queue {
359359 // If we're complete, then there's nothing to do, we just
360360 // jettison out as we shouldn't run the closure.
@@ -401,33 +401,45 @@ impl Once {
401401 // not RUNNING.
402402 _ => {
403403 assert ! ( state_and_queue & STATE_MASK == RUNNING ) ;
404+ // Create the node for our current thread that we are going to try to slot
405+ // in at the head of the linked list.
404406 let mut node = Waiter {
405407 thread : Some ( thread:: current ( ) ) ,
406408 signaled : AtomicBool :: new ( false ) ,
407409 next : ptr:: null_mut ( ) ,
408410 } ;
409411 let me = & mut node as * mut Waiter as usize ;
410- assert ! ( me & STATE_MASK == 0 ) ;
412+ assert ! ( me & STATE_MASK == 0 ) ; // We assume pointers have 2 free bits that
413+ // we can use for state.
414+
415+ // Try to slide in the node at the head of the linked list.
416+ // Run in a loop where we make sure the status is still RUNNING, and that
417+ // another thread did not just replace the head of the linked list.
418+ let mut old_head_and_status = state_and_queue;
419+ loop {
420+ if old_head_and_status & STATE_MASK != RUNNING {
421+ return ; // No need anymore to enqueue ourselves.
422+ }
411423
412- while state_and_queue & STATE_MASK == RUNNING {
413- node. next = ( state_and_queue & !STATE_MASK ) as * mut Waiter ;
414- let old = self . state_and_queue . compare_and_swap ( state_and_queue,
424+ node. next = ( old_head_and_status & !STATE_MASK ) as * mut Waiter ;
425+ let old = self . state_and_queue . compare_and_swap ( old_head_and_status,
415426 me | RUNNING ,
416- Ordering :: SeqCst ) ;
417- if old != state_and_queue {
418- state_and_queue = old;
419- continue
427+ Ordering :: Release ) ;
428+ if old == old_head_and_status {
429+ break ; // Success!
420430 }
431+ old_head_and_status = old;
432+ }
421433
422- // Once we've enqueued ourselves, wait in a loop.
423- // Afterwards reload the state and continue with what we
424- // were doing from before.
425- while !node. signaled . load ( Ordering :: SeqCst ) {
426- thread:: park ( ) ;
427- }
428- state_and_queue = self . state_and_queue . load ( Ordering :: SeqCst ) ;
429- continue ' outer
434+ // We have enqueued ourselves, now lets wait.
435+ // It is important not to return before being signaled, otherwise we would
436+ // drop our `Waiter` node and leave a hole in the linked list (and a
437+ // dangling reference). Guard against spurious wakeups by reparking
438+ // ourselves until we are signaled.
439+ while !node. signaled . load ( Ordering :: SeqCst ) {
440+ thread:: park ( ) ;
430441 }
442+ state_and_queue = self . state_and_queue . load ( Ordering :: SeqCst ) ;
431443 }
432444 }
433445 }
0 commit comments