@@ -2284,18 +2284,14 @@ event_sched_out(struct perf_event *event, struct perf_event_context *ctx)
22842284 }
22852285
22862286 if (event -> pending_sigtrap ) {
2287- bool dec = true;
2288-
22892287 event -> pending_sigtrap = 0 ;
22902288 if (state != PERF_EVENT_STATE_OFF &&
2291- !event -> pending_work ) {
2289+ !event -> pending_work &&
2290+ !task_work_add (current , & event -> pending_task , TWA_RESUME )) {
22922291 event -> pending_work = 1 ;
2293- dec = false;
2294- WARN_ON_ONCE (!atomic_long_inc_not_zero (& event -> refcount ));
2295- task_work_add (current , & event -> pending_task , TWA_RESUME );
2296- }
2297- if (dec )
2292+ } else {
22982293 local_dec (& event -> ctx -> nr_pending );
2294+ }
22992295 }
23002296
23012297 perf_event_set_state (event , state );
@@ -5187,9 +5183,35 @@ static bool exclusive_event_installable(struct perf_event *event,
51875183static void perf_addr_filters_splice (struct perf_event * event ,
51885184 struct list_head * head );
51895185
5186+ static void perf_pending_task_sync (struct perf_event * event )
5187+ {
5188+ struct callback_head * head = & event -> pending_task ;
5189+
5190+ if (!event -> pending_work )
5191+ return ;
5192+ /*
5193+ * If the task is queued to the current task's queue, we
5194+ * obviously can't wait for it to complete. Simply cancel it.
5195+ */
5196+ if (task_work_cancel (current , head )) {
5197+ event -> pending_work = 0 ;
5198+ local_dec (& event -> ctx -> nr_pending );
5199+ return ;
5200+ }
5201+
5202+ /*
5203+ * All accesses related to the event are within the same
5204+ * non-preemptible section in perf_pending_task(). The RCU
5205+ * grace period before the event is freed will make sure all
5206+ * those accesses are complete by then.
5207+ */
5208+ rcuwait_wait_event (& event -> pending_work_wait , !event -> pending_work , TASK_UNINTERRUPTIBLE );
5209+ }
5210+
51905211static void _free_event (struct perf_event * event )
51915212{
51925213 irq_work_sync (& event -> pending_irq );
5214+ perf_pending_task_sync (event );
51935215
51945216 unaccount_event (event );
51955217
@@ -6807,24 +6829,28 @@ static void perf_pending_task(struct callback_head *head)
68076829 struct perf_event * event = container_of (head , struct perf_event , pending_task );
68086830 int rctx ;
68096831
6832+ /*
6833+ * All accesses to the event must belong to the same implicit RCU read-side
6834+ * critical section as the ->pending_work reset. See comment in
6835+ * perf_pending_task_sync().
6836+ */
6837+ preempt_disable_notrace ();
68106838 /*
68116839 * If we 'fail' here, that's OK, it means recursion is already disabled
68126840 * and we won't recurse 'further'.
68136841 */
6814- preempt_disable_notrace ();
68156842 rctx = perf_swevent_get_recursion_context ();
68166843
68176844 if (event -> pending_work ) {
68186845 event -> pending_work = 0 ;
68196846 perf_sigtrap (event );
68206847 local_dec (& event -> ctx -> nr_pending );
6848+ rcuwait_wake_up (& event -> pending_work_wait );
68216849 }
68226850
68236851 if (rctx >= 0 )
68246852 perf_swevent_put_recursion_context (rctx );
68256853 preempt_enable_notrace ();
6826-
6827- put_event (event );
68286854}
68296855
68306856#ifdef CONFIG_GUEST_PERF_EVENTS
@@ -11939,6 +11965,7 @@ perf_event_alloc(struct perf_event_attr *attr, int cpu,
1193911965 init_waitqueue_head (& event -> waitq );
1194011966 init_irq_work (& event -> pending_irq , perf_pending_irq );
1194111967 init_task_work (& event -> pending_task , perf_pending_task );
11968+ rcuwait_init (& event -> pending_work_wait );
1194211969
1194311970 mutex_init (& event -> mmap_mutex );
1194411971 raw_spin_lock_init (& event -> addr_filters .lock );
0 commit comments