Line data    Source code

       1             : #include <linux/spinlock.h>
       2             : #include <linux/task_work.h>
       3             : #include <linux/tracehook.h>
       4             : 
       5             : static struct callback_head work_exited; /* all we need is ->next == NULL */
       6             : 
       7             : /**
       8             :  * task_work_add - ask the @task to execute @work->func()
       9             :  * @task: the task which should run the callback
      10             :  * @work: the callback to run
      11             :  * @notify: send the notification if true
      12             :  *
      13             :  * Queue @work for task_work_run() below and notify the @task if @notify.
      14             :  * Fails if the @task is exiting/exited and thus it can't process this @work.
      15             :  * Otherwise @work->func() will be called when the @task returns from kernel
      16             :  * mode or exits.
      17             :  *
      18             :  * This is like the signal handler which runs in kernel mode, but it doesn't
      19             :  * try to wake up the @task.
      20             :  *
      21             :  * RETURNS:
      22             :  * 0 if succeeds or -ESRCH.
      23             :  */
      24             : int
      25       46953 : task_work_add(struct task_struct *task, struct callback_head *work, bool notify)
      26             : {
      27             :         struct callback_head *head;
      28             : 
      29             :         do {
      30       46953 :                 head = ACCESS_ONCE(task->task_works);
      31       46953 :                 if (unlikely(head == &work_exited))
      32             :                         return -ESRCH;
      33       46953 :                 work->next = head;
      34       93906 :         } while (cmpxchg(&task->task_works, head, work) != head);
      35             : 
      36       46953 :         if (notify)
      37             :                 set_notify_resume(task);
      38             :         return 0;
      39             : }
      40             : 
      41             : /**
      42             :  * task_work_cancel - cancel a pending work added by task_work_add()
      43             :  * @task: the task which should execute the work
      44             :  * @func: identifies the work to remove
      45             :  *
      46             :  * Find the last queued pending work with ->func == @func and remove
      47             :  * it from queue.
      48             :  *
      49             :  * RETURNS:
      50             :  * The found work or NULL if not found.
      51             :  */
      52             : struct callback_head *
      53           0 : task_work_cancel(struct task_struct *task, task_work_func_t func)
      54             : {
      55           0 :         struct callback_head **pprev = &task->task_works;
      56             :         struct callback_head *work;
      57             :         unsigned long flags;
      58             :         /*
      59             :          * If cmpxchg() fails we continue without updating pprev.
      60             :          * Either we raced with task_work_add() which added the
      61             :          * new entry before this work, we will find it again. Or
      62             :          * we raced with task_work_run(), *pprev == NULL/exited.
      63             :          */
      64           0 :         raw_spin_lock_irqsave(&task->pi_lock, flags);
      65           0 :         while ((work = ACCESS_ONCE(*pprev))) {
      66             :                 smp_read_barrier_depends();
      67           0 :                 if (work->func != func)
      68           0 :                         pprev = &work->next;
      69           0 :                 else if (cmpxchg(pprev, work, work->next) == work)
      70             :                         break;
      71             :         }
      72           0 :         raw_spin_unlock_irqrestore(&task->pi_lock, flags);
      73             : 
      74           0 :         return work;
      75             : }
      76             : 
      77             : /**
      78             :  * task_work_run - execute the works added by task_work_add()
      79             :  *
      80             :  * Flush the pending works. Should be used by the core kernel code.
      81             :  * Called before the task returns to the user-mode or stops, or when
      82             :  * it exits. In the latter case task_work_add() can no longer add the
      83             :  * new work after task_work_run() returns.
      84             :  */
      85       35170 : void task_work_run(void)
      86             : {
      87       35170 :         struct task_struct *task = current;
      88             :         struct callback_head *work, *head, *next;
      89             : 
      90             :         for (;;) {
      91             :                 /*
      92             :                  * work->func() can do task_work_add(), do not set
      93             :                  * work_exited unless the list is empty.
      94             :                  */
      95             :                 do {
      96       70210 :                         work = ACCESS_ONCE(task->task_works);
      97       35170 :                         head = !work && (task->flags & PF_EXITING) ?
      98       70210 :                                 &work_exited : NULL;
      99      140420 :                 } while (cmpxchg(&task->task_works, work, head) != work);
     100             : 
     101       70210 :                 if (!work)
     102             :                         break;
     103             :                 /*
     104             :                  * Synchronize with task_work_cancel(). It can't remove
     105             :                  * the first entry == work, cmpxchg(task_works) should
     106             :                  * fail, but it can play with *work and other entries.
     107             :                  */
     108       35040 :                 raw_spin_unlock_wait(&task->pi_lock);
     109       35040 :                 smp_mb();
     110             : 
     111             :                 /* Reverse the list to run the works in fifo order */
     112             :                 head = NULL;
     113             :                 do {
     114       46953 :                         next = work->next;
     115       46953 :                         work->next = head;
     116             :                         head = work;
     117             :                         work = next;
     118       46953 :                 } while (work);
     119             : 
     120             :                 work = head;
     121             :                 do {
     122       46953 :                         next = work->next;
     123       46953 :                         work->func(work);
     124             :                         work = next;
     125       46951 :                         cond_resched();
     126       46953 :                 } while (work);
     127             :         }
     128       35170 : }