arch/s390/kernel/vtime.c

   1 /*
   2  *    Virtual cpu timer based timer functions.
   3  *
   4  *    Copyright IBM Corp. 2004, 2012
   5  *    Author(s): Jan Glauber <jan.glauber@de.ibm.com>
   6  */
   7
   8 #include <linux/kernel_stat.h>
   9 #include <linux/sched/cputime.h>
  10 #include <linux/export.h>
  11 #include <linux/kernel.h>
  12 #include <linux/timex.h>
  13 #include <linux/types.h>
  14 #include <linux/time.h>
  15
  16 #include <asm/vtimer.h>
  17 #include <asm/vtime.h>
  18 #include <asm/cpu_mf.h>
  19 #include <asm/smp.h>
  20
  21 #include "entry.h"
  22
  23 static void virt_timer_expire(void);
  24
  25 static LIST_HEAD(virt_timer_list);
  26 static DEFINE_SPINLOCK(virt_timer_lock);
  27 static atomic64_t virt_timer_current;
  28 static atomic64_t virt_timer_elapsed;
  29
  30 DEFINE_PER_CPU(u64, mt_cycles[8]);
  31 static DEFINE_PER_CPU(u64, mt_scaling_mult) = { 1 };
  32 static DEFINE_PER_CPU(u64, mt_scaling_div) = { 1 };
  33 static DEFINE_PER_CPU(u64, mt_scaling_jiffies);
  34
  35 static inline u64 get_vtimer(void)
  36 {
  37         u64 timer;
  38
  39         asm volatile("stpt %0" : "=m" (timer));
  40         return timer;
  41 }
  42
  43 static inline void set_vtimer(u64 expires)
  44 {
  45         u64 timer;
  46
  47         asm volatile(
  48                 "       stpt    %0\n"   /* Store current cpu timer value */
  49                 "       spt     %1"     /* Set new value imm. afterwards */
  50                 : "=m" (timer) : "m" (expires));
  51         S390_lowcore.system_timer += S390_lowcore.last_update_timer - timer;
  52         S390_lowcore.last_update_timer = expires;
  53 }
  54
  55 static inline int virt_timer_forward(u64 elapsed)
  56 {
  57         BUG_ON(!irqs_disabled());
  58
  59         if (list_empty(&virt_timer_list))
  60                 return 0;
  61         elapsed = atomic64_add_return(elapsed, &virt_timer_elapsed);
  62         return elapsed >= atomic64_read(&virt_timer_current);
  63 }
  64
  65 static void update_mt_scaling(void)
  66 {
  67         u64 cycles_new[8], *cycles_old;
  68         u64 delta, fac, mult, div;
  69         int i;
  70
  71         stcctm5(smp_cpu_mtid + 1, cycles_new);
  72         cycles_old = this_cpu_ptr(mt_cycles);
  73         fac = 1;
  74         mult = div = 0;
  75         for (i = 0; i <= smp_cpu_mtid; i++) {
  76                 delta = cycles_new[i] - cycles_old[i];
  77                 div += delta;
  78                 mult *= i + 1;
  79                 mult += delta * fac;
  80                 fac *= i + 1;
  81         }
  82         div *= fac;
  83         if (div > 0) {
  84                 /* Update scaling factor */
  85                 __this_cpu_write(mt_scaling_mult, mult);
  86                 __this_cpu_write(mt_scaling_div, div);
  87                 memcpy(cycles_old, cycles_new,
  88                        sizeof(u64) * (smp_cpu_mtid + 1));
  89         }
  90         __this_cpu_write(mt_scaling_jiffies, jiffies_64);
  91 }
  92
  93 static inline u64 update_tsk_timer(unsigned long *tsk_vtime, u64 new)
  94 {
  95         u64 delta;
  96
  97         delta = new - *tsk_vtime;
  98         *tsk_vtime = new;
  99         return delta;
 100 }
 101
 102
 103 static inline u64 scale_vtime(u64 vtime)
 104 {
 105         u64 mult = __this_cpu_read(mt_scaling_mult);
 106         u64 div = __this_cpu_read(mt_scaling_div);
 107
 108         if (smp_cpu_mtid)
 109                 return vtime * mult / div;
 110         return vtime;
 111 }
 112
 113 static void account_system_index_scaled(struct task_struct *p, u64 cputime,
 114                                         enum cpu_usage_stat index)
 115 {
 116         p->stimescaled += cputime_to_nsecs(scale_vtime(cputime));
 117         account_system_index_time(p, cputime_to_nsecs(cputime), index);
 118 }
 119
 120 /*
 121  * Update process times based on virtual cpu times stored by entry.S
 122  * to the lowcore fields user_timer, system_timer & steal_clock.
 123  */
 124 static int do_account_vtime(struct task_struct *tsk)
 125 {
 126         u64 timer, clock, user, guest, system, hardirq, softirq, steal;
 127
 128         timer = S390_lowcore.last_update_timer;
 129         clock = S390_lowcore.last_update_clock;
 130         asm volatile(
 131                 "       stpt    %0\n"   /* Store current cpu timer value */
 132 #ifdef CONFIG_HAVE_MARCH_Z9_109_FEATURES
 133                 "       stckf   %1"     /* Store current tod clock value */
 134 #else
 135                 "       stck    %1"     /* Store current tod clock value */
 136 #endif
 137                 : "=m" (S390_lowcore.last_update_timer),
 138                   "=m" (S390_lowcore.last_update_clock));
 139         clock = S390_lowcore.last_update_clock - clock;
 140         timer -= S390_lowcore.last_update_timer;
 141
 142         if (hardirq_count())
 143                 S390_lowcore.hardirq_timer += timer;
 144         else
 145                 S390_lowcore.system_timer += timer;
 146
 147         /* Update MT utilization calculation */
 148         if (smp_cpu_mtid &&
 149             time_after64(jiffies_64, this_cpu_read(mt_scaling_jiffies)))
 150                 update_mt_scaling();
 151
 152         /* Calculate cputime delta */
 153         user = update_tsk_timer(&tsk->thread.user_timer,
 154                                 READ_ONCE(S390_lowcore.user_timer));
 155         guest = update_tsk_timer(&tsk->thread.guest_timer,
 156                                  READ_ONCE(S390_lowcore.guest_timer));
 157         system = update_tsk_timer(&tsk->thread.system_timer,
 158                                   READ_ONCE(S390_lowcore.system_timer));
 159         hardirq = update_tsk_timer(&tsk->thread.hardirq_timer,
 160                                    READ_ONCE(S390_lowcore.hardirq_timer));
 161         softirq = update_tsk_timer(&tsk->thread.softirq_timer,
 162                                    READ_ONCE(S390_lowcore.softirq_timer));
 163         S390_lowcore.steal_timer +=
 164                 clock - user - guest - system - hardirq - softirq;
 165
 166         /* Push account value */
 167         if (user) {
 168                 account_user_time(tsk, cputime_to_nsecs(user));
 169                 tsk->utimescaled += cputime_to_nsecs(scale_vtime(user));
 170         }
 171
 172         if (guest) {
 173                 account_guest_time(tsk, cputime_to_nsecs(guest));
 174                 tsk->utimescaled += cputime_to_nsecs(scale_vtime(guest));
 175         }
 176
 177         if (system)
 178                 account_system_index_scaled(tsk, system, CPUTIME_SYSTEM);
 179         if (hardirq)
 180                 account_system_index_scaled(tsk, hardirq, CPUTIME_IRQ);
 181         if (softirq)
 182                 account_system_index_scaled(tsk, softirq, CPUTIME_SOFTIRQ);
 183
 184         steal = S390_lowcore.steal_timer;
 185         if ((s64) steal > 0) {
 186                 S390_lowcore.steal_timer = 0;
 187                 account_steal_time(cputime_to_nsecs(steal));
 188         }
 189
 190         return virt_timer_forward(user + guest + system + hardirq + softirq);
 191 }
 192
 193 void vtime_task_switch(struct task_struct *prev)
 194 {
 195         do_account_vtime(prev);
 196         prev->thread.user_timer = S390_lowcore.user_timer;
 197         prev->thread.guest_timer = S390_lowcore.guest_timer;
 198         prev->thread.system_timer = S390_lowcore.system_timer;
 199         prev->thread.hardirq_timer = S390_lowcore.hardirq_timer;
 200         prev->thread.softirq_timer = S390_lowcore.softirq_timer;
 201         S390_lowcore.user_timer = current->thread.user_timer;
 202         S390_lowcore.guest_timer = current->thread.guest_timer;
 203         S390_lowcore.system_timer = current->thread.system_timer;
 204         S390_lowcore.hardirq_timer = current->thread.hardirq_timer;
 205         S390_lowcore.softirq_timer = current->thread.softirq_timer;
 206 }
 207
 208 /*
 209  * In s390, accounting pending user time also implies
 210  * accounting system time in order to correctly compute
 211  * the stolen time accounting.
 212  */
 213 void vtime_flush(struct task_struct *tsk)
 214 {
 215         if (do_account_vtime(tsk))
 216                 virt_timer_expire();
 217 }
 218
 219 /*
 220  * Update process times based on virtual cpu times stored by entry.S
 221  * to the lowcore fields user_timer, system_timer & steal_clock.
 222  */
 223 void vtime_account_irq_enter(struct task_struct *tsk)
 224 {
 225         u64 timer;
 226
 227         timer = S390_lowcore.last_update_timer;
 228         S390_lowcore.last_update_timer = get_vtimer();
 229         timer -= S390_lowcore.last_update_timer;
 230
 231         if ((tsk->flags & PF_VCPU) && (irq_count() == 0))
 232                 S390_lowcore.guest_timer += timer;
 233         else if (hardirq_count())
 234                 S390_lowcore.hardirq_timer += timer;
 235         else if (in_serving_softirq())
 236                 S390_lowcore.softirq_timer += timer;
 237         else
 238                 S390_lowcore.system_timer += timer;
 239
 240         virt_timer_forward(timer);
 241 }
 242 EXPORT_SYMBOL_GPL(vtime_account_irq_enter);
 243
 244 void vtime_account_system(struct task_struct *tsk)
 245 __attribute__((alias("vtime_account_irq_enter")));
 246 EXPORT_SYMBOL_GPL(vtime_account_system);
 247
 248 /*
 249  * Sorted add to a list. List is linear searched until first bigger
 250  * element is found.
 251  */
 252 static void list_add_sorted(struct vtimer_list *timer, struct list_head *head)
 253 {
 254         struct vtimer_list *tmp;
 255
 256         list_for_each_entry(tmp, head, entry) {
 257                 if (tmp->expires > timer->expires) {
 258                         list_add_tail(&timer->entry, &tmp->entry);
 259                         return;
 260                 }
 261         }
 262         list_add_tail(&timer->entry, head);
 263 }
 264
 265 /*
 266  * Handler for expired virtual CPU timer.
 267  */
 268 static void virt_timer_expire(void)
 269 {
 270         struct vtimer_list *timer, *tmp;
 271         unsigned long elapsed;
 272         LIST_HEAD(cb_list);
 273
 274         /* walk timer list, fire all expired timers */
 275         spin_lock(&virt_timer_lock);
 276         elapsed = atomic64_read(&virt_timer_elapsed);
 277         list_for_each_entry_safe(timer, tmp, &virt_timer_list, entry) {
 278                 if (timer->expires < elapsed)
 279                         /* move expired timer to the callback queue */
 280                         list_move_tail(&timer->entry, &cb_list);
 281                 else
 282                         timer->expires -= elapsed;
 283         }
 284         if (!list_empty(&virt_timer_list)) {
 285                 timer = list_first_entry(&virt_timer_list,
 286                                          struct vtimer_list, entry);
 287                 atomic64_set(&virt_timer_current, timer->expires);
 288         }
 289         atomic64_sub(elapsed, &virt_timer_elapsed);
 290         spin_unlock(&virt_timer_lock);
 291
 292         /* Do callbacks and recharge periodic timers */
 293         list_for_each_entry_safe(timer, tmp, &cb_list, entry) {
 294                 list_del_init(&timer->entry);
 295                 timer->function(timer->data);
 296                 if (timer->interval) {
 297                         /* Recharge interval timer */
 298                         timer->expires = timer->interval +
 299                                 atomic64_read(&virt_timer_elapsed);
 300                         spin_lock(&virt_timer_lock);
 301                         list_add_sorted(timer, &virt_timer_list);
 302                         spin_unlock(&virt_timer_lock);
 303                 }
 304         }
 305 }
 306
 307 void init_virt_timer(struct vtimer_list *timer)
 308 {
 309         timer->function = NULL;
 310         INIT_LIST_HEAD(&timer->entry);
 311 }
 312 EXPORT_SYMBOL(init_virt_timer);
 313
 314 static inline int vtimer_pending(struct vtimer_list *timer)
 315 {
 316         return !list_empty(&timer->entry);
 317 }
 318
 319 static void internal_add_vtimer(struct vtimer_list *timer)
 320 {
 321         if (list_empty(&virt_timer_list)) {
 322                 /* First timer, just program it. */
 323                 atomic64_set(&virt_timer_current, timer->expires);
 324                 atomic64_set(&virt_timer_elapsed, 0);
 325                 list_add(&timer->entry, &virt_timer_list);
 326         } else {
 327                 /* Update timer against current base. */
 328                 timer->expires += atomic64_read(&virt_timer_elapsed);
 329                 if (likely((s64) timer->expires <
 330                            (s64) atomic64_read(&virt_timer_current)))
 331                         /* The new timer expires before the current timer. */
 332                         atomic64_set(&virt_timer_current, timer->expires);
 333                 /* Insert new timer into the list. */
 334                 list_add_sorted(timer, &virt_timer_list);
 335         }
 336 }
 337
 338 static void __add_vtimer(struct vtimer_list *timer, int periodic)
 339 {
 340         unsigned long flags;
 341
 342         timer->interval = periodic ? timer->expires : 0;
 343         spin_lock_irqsave(&virt_timer_lock, flags);
 344         internal_add_vtimer(timer);
 345         spin_unlock_irqrestore(&virt_timer_lock, flags);
 346 }
 347
 348 /*
 349  * add_virt_timer - add a oneshot virtual CPU timer
 350  */
 351 void add_virt_timer(struct vtimer_list *timer)
 352 {
 353         __add_vtimer(timer, 0);
 354 }
 355 EXPORT_SYMBOL(add_virt_timer);
 356
 357 /*
 358  * add_virt_timer_int - add an interval virtual CPU timer
 359  */
 360 void add_virt_timer_periodic(struct vtimer_list *timer)
 361 {
 362         __add_vtimer(timer, 1);
 363 }
 364 EXPORT_SYMBOL(add_virt_timer_periodic);
 365
 366 static int __mod_vtimer(struct vtimer_list *timer, u64 expires, int periodic)
 367 {
 368         unsigned long flags;
 369         int rc;
 370
 371         BUG_ON(!timer->function);
 372
 373         if (timer->expires == expires && vtimer_pending(timer))
 374                 return 1;
 375         spin_lock_irqsave(&virt_timer_lock, flags);
 376         rc = vtimer_pending(timer);
 377         if (rc)
 378                 list_del_init(&timer->entry);
 379         timer->interval = periodic ? expires : 0;
 380         timer->expires = expires;
 381         internal_add_vtimer(timer);
 382         spin_unlock_irqrestore(&virt_timer_lock, flags);
 383         return rc;
 384 }
 385
 386 /*
 387  * returns whether it has modified a pending timer (1) or not (0)
 388  */
 389 int mod_virt_timer(struct vtimer_list *timer, u64 expires)
 390 {
 391         return __mod_vtimer(timer, expires, 0);
 392 }
 393 EXPORT_SYMBOL(mod_virt_timer);
 394
 395 /*
 396  * returns whether it has modified a pending timer (1) or not (0)
 397  */
 398 int mod_virt_timer_periodic(struct vtimer_list *timer, u64 expires)
 399 {
 400         return __mod_vtimer(timer, expires, 1);
 401 }
 402 EXPORT_SYMBOL(mod_virt_timer_periodic);
 403
 404 /*
 405  * Delete a virtual timer.
 406  *
 407  * returns whether the deleted timer was pending (1) or not (0)
 408  */
 409 int del_virt_timer(struct vtimer_list *timer)
 410 {
 411         unsigned long flags;
 412
 413         if (!vtimer_pending(timer))
 414                 return 0;
 415         spin_lock_irqsave(&virt_timer_lock, flags);
 416         list_del_init(&timer->entry);
 417         spin_unlock_irqrestore(&virt_timer_lock, flags);
 418         return 1;
 419 }
 420 EXPORT_SYMBOL(del_virt_timer);
 421
 422 /*
 423  * Start the virtual CPU timer on the current CPU.
 424  */
 425 void vtime_init(void)
 426 {
 427         /* set initial cpu timer */
 428         set_vtimer(VTIMER_MAX_SLICE);
 429         /* Setup initial MT scaling values */
 430         if (smp_cpu_mtid) {
 431                 __this_cpu_write(mt_scaling_jiffies, jiffies);
 432                 __this_cpu_write(mt_scaling_mult, 1);
 433                 __this_cpu_write(mt_scaling_div, 1);
 434                 stcctm5(smp_cpu_mtid + 1, this_cpu_ptr(mt_cycles));
 435         }
 436 }