arch/riscv/kernel/patch.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (C) 2020 SiFive
   4  */
   5
   6 #include <linux/spinlock.h>
   7 #include <linux/mm.h>
   8 #include <linux/uaccess.h>
   9 #include <linux/stop_machine.h>
  10 #include <asm/kprobes.h>
  11 #include <asm/cacheflush.h>
  12 #include <asm/fixmap.h>
  13
  14 struct riscv_insn_patch {
  15         void *addr;
  16         u32 insn;
  17         atomic_t cpu_count;
  18 };
  19
  20 #ifdef CONFIG_MMU
  21 static DEFINE_RAW_SPINLOCK(patch_lock);
  22
  23 static void __kprobes *patch_map(void *addr, int fixmap)
  24 {
  25         uintptr_t uintaddr = (uintptr_t) addr;
  26         struct page *page;
  27
  28         if (core_kernel_text(uintaddr))
  29                 page = phys_to_page(__pa_symbol(addr));
  30         else if (IS_ENABLED(CONFIG_STRICT_MODULE_RWX))
  31                 page = vmalloc_to_page(addr);
  32         else
  33                 return addr;
  34
  35         BUG_ON(!page);
  36
  37         return (void *)set_fixmap_offset(fixmap, page_to_phys(page) +
  38                                          (uintaddr & ~PAGE_MASK));
  39 }
  40
  41 static void __kprobes patch_unmap(int fixmap)
  42 {
  43         clear_fixmap(fixmap);
  44 }
  45
  46 static int __kprobes riscv_insn_write(void *addr, const void *insn, size_t len)
  47 {
  48         void *waddr = addr;
  49         bool across_pages = (((uintptr_t) addr & ~PAGE_MASK) + len) > PAGE_SIZE;
  50         unsigned long flags = 0;
  51         int ret;
  52
  53         raw_spin_lock_irqsave(&patch_lock, flags);
  54
  55         if (across_pages)
  56                 patch_map(addr + len, FIX_TEXT_POKE1);
  57
  58         waddr = patch_map(addr, FIX_TEXT_POKE0);
  59
  60         ret = probe_kernel_write(waddr, insn, len);
  61
  62         patch_unmap(FIX_TEXT_POKE0);
  63
  64         if (across_pages)
  65                 patch_unmap(FIX_TEXT_POKE1);
  66
  67         raw_spin_unlock_irqrestore(&patch_lock, flags);
  68
  69         return ret;
  70 }
  71 #else
  72 static int __kprobes riscv_insn_write(void *addr, const void *insn, size_t len)
  73 {
  74         return probe_kernel_write(addr, insn, len);
  75 }
  76 #endif /* CONFIG_MMU */
  77
  78 int __kprobes riscv_patch_text_nosync(void *addr, const void *insns, size_t len)
  79 {
  80         u32 *tp = addr;
  81         int ret;
  82
  83         ret = riscv_insn_write(tp, insns, len);
  84
  85         if (!ret)
  86                 flush_icache_range((uintptr_t) tp, (uintptr_t) tp + len);
  87
  88         return ret;
  89 }
  90
  91 static int __kprobes riscv_patch_text_cb(void *data)
  92 {
  93         struct riscv_insn_patch *patch = data;
  94         int ret = 0;
  95
  96         if (atomic_inc_return(&patch->cpu_count) == 1) {
  97                 ret =
  98                     riscv_patch_text_nosync(patch->addr, &patch->insn,
  99                                             GET_INSN_LENGTH(patch->insn));
 100                 atomic_inc(&patch->cpu_count);
 101         } else {
 102                 while (atomic_read(&patch->cpu_count) <= num_online_cpus())
 103                         cpu_relax();
 104                 smp_mb();
 105         }
 106
 107         return ret;
 108 }
 109
 110 int __kprobes riscv_patch_text(void *addr, u32 insn)
 111 {
 112         struct riscv_insn_patch patch = {
 113                 .addr = addr,
 114                 .insn = insn,
 115                 .cpu_count = ATOMIC_INIT(0),
 116         };
 117
 118         return stop_machine_cpuslocked(riscv_patch_text_cb,
 119                                        &patch, cpu_online_mask);
 120 }