arch/mips/include/asm/uaccess.h

   1 /*
   2  * This file is subject to the terms and conditions of the GNU General Public
   3  * License.  See the file "COPYING" in the main directory of this archive
   4  * for more details.
   5  *
   6  * Copyright (C) 1996, 1997, 1998, 1999, 2000, 03, 04 by Ralf Baechle
   7  * Copyright (C) 1999, 2000 Silicon Graphics, Inc.
   8  * Copyright (C) 2007  Maciej W. Rozycki
   9  * Copyright (C) 2014, Imagination Technologies Ltd.
  10  */
  11 #ifndef _ASM_UACCESS_H
  12 #define _ASM_UACCESS_H
  13
  14 #include <linux/kernel.h>
  15 #include <linux/string.h>
  16 #include <asm/asm-eva.h>
  17 #include <asm/extable.h>
  18
  19 /*
  20  * The fs value determines whether argument validity checking should be
  21  * performed or not.  If get_fs() == USER_DS, checking is performed, with
  22  * get_fs() == KERNEL_DS, checking is bypassed.
  23  *
  24  * For historical reasons, these macros are grossly misnamed.
  25  */
  26 #ifdef CONFIG_32BIT
  27
  28 #ifdef CONFIG_KVM_GUEST
  29 #define __UA_LIMIT 0x40000000UL
  30 #else
  31 #define __UA_LIMIT 0x80000000UL
  32 #endif
  33
  34 #define __UA_ADDR       ".word"
  35 #define __UA_LA         "la"
  36 #define __UA_ADDU       "addu"
  37 #define __UA_t0         "$8"
  38 #define __UA_t1         "$9"
  39
  40 #endif /* CONFIG_32BIT */
  41
  42 #ifdef CONFIG_64BIT
  43
  44 extern u64 __ua_limit;
  45
  46 #define __UA_LIMIT      __ua_limit
  47
  48 #define __UA_ADDR       ".dword"
  49 #define __UA_LA         "dla"
  50 #define __UA_ADDU       "daddu"
  51 #define __UA_t0         "$12"
  52 #define __UA_t1         "$13"
  53
  54 #endif /* CONFIG_64BIT */
  55
  56 /*
  57  * USER_DS is a bitmask that has the bits set that may not be set in a valid
  58  * userspace address.  Note that we limit 32-bit userspace to 0x7fff8000 but
  59  * the arithmetic we're doing only works if the limit is a power of two, so
  60  * we use 0x80000000 here on 32-bit kernels.  If a process passes an invalid
  61  * address in this range it's the process's problem, not ours :-)
  62  */
  63
  64 #ifdef CONFIG_KVM_GUEST
  65 #define KERNEL_DS       ((mm_segment_t) { 0x80000000UL })
  66 #define USER_DS         ((mm_segment_t) { 0xC0000000UL })
  67 #else
  68 #define KERNEL_DS       ((mm_segment_t) { 0UL })
  69 #define USER_DS         ((mm_segment_t) { __UA_LIMIT })
  70 #endif
  71
  72 #define get_fs()        (current_thread_info()->addr_limit)
  73 #define set_fs(x)       (current_thread_info()->addr_limit = (x))
  74
  75 #define segment_eq(a, b)        ((a).seg == (b).seg)
  76
  77 /*
  78  * eva_kernel_access() - determine whether kernel memory access on an EVA system
  79  *
  80  * Determines whether memory accesses should be performed to kernel memory
  81  * on a system using Extended Virtual Addressing (EVA).
  82  *
  83  * Return: true if a kernel memory access on an EVA system, else false.
  84  */
  85 static inline bool eva_kernel_access(void)
  86 {
  87         if (!IS_ENABLED(CONFIG_EVA))
  88                 return false;
  89
  90         return uaccess_kernel();
  91 }
  92
  93 /*
  94  * Is a address valid? This does a straightforward calculation rather
  95  * than tests.
  96  *
  97  * Address valid if:
  98  *  - "addr" doesn't have any high-bits set
  99  *  - AND "size" doesn't have any high-bits set
 100  *  - AND "addr+size" doesn't have any high-bits set
 101  *  - OR we are in kernel mode.
 102  *
 103  * __ua_size() is a trick to avoid runtime checking of positive constant
 104  * sizes; for those we already know at compile time that the size is ok.
 105  */
 106 #define __ua_size(size)                                                 \
 107         ((__builtin_constant_p(size) && (signed long) (size) > 0) ? 0 : (size))
 108
 109 /*
 110  * access_ok: - Checks if a user space pointer is valid
 111  * @addr: User space pointer to start of block to check
 112  * @size: Size of block to check
 113  *
 114  * Context: User context only. This function may sleep if pagefaults are
 115  *          enabled.
 116  *
 117  * Checks if a pointer to a block of memory in user space is valid.
 118  *
 119  * Returns true (nonzero) if the memory block may be valid, false (zero)
 120  * if it is definitely invalid.
 121  *
 122  * Note that, depending on architecture, this function probably just
 123  * checks that the pointer is in the user space range - after calling
 124  * this function, memory access functions may still return -EFAULT.
 125  */
 126
 127 static inline int __access_ok(const void __user *p, unsigned long size)
 128 {
 129         unsigned long addr = (unsigned long)p;
 130         return (get_fs().seg & (addr | (addr + size) | __ua_size(size))) == 0;
 131 }
 132
 133 #define access_ok(addr, size)                                   \
 134         likely(__access_ok((addr), (size)))
 135
 136 /*
 137  * put_user: - Write a simple value into user space.
 138  * @x:   Value to copy to user space.
 139  * @ptr: Destination address, in user space.
 140  *
 141  * Context: User context only. This function may sleep if pagefaults are
 142  *          enabled.
 143  *
 144  * This macro copies a single simple value from kernel space to user
 145  * space.  It supports simple types like char and int, but not larger
 146  * data types like structures or arrays.
 147  *
 148  * @ptr must have pointer-to-simple-variable type, and @x must be assignable
 149  * to the result of dereferencing @ptr.
 150  *
 151  * Returns zero on success, or -EFAULT on error.
 152  */
 153 #define put_user(x,ptr) \
 154         __put_user_check((x), (ptr), sizeof(*(ptr)))
 155
 156 /*
 157  * get_user: - Get a simple variable from user space.
 158  * @x:   Variable to store result.
 159  * @ptr: Source address, in user space.
 160  *
 161  * Context: User context only. This function may sleep if pagefaults are
 162  *          enabled.
 163  *
 164  * This macro copies a single simple variable from user space to kernel
 165  * space.  It supports simple types like char and int, but not larger
 166  * data types like structures or arrays.
 167  *
 168  * @ptr must have pointer-to-simple-variable type, and the result of
 169  * dereferencing @ptr must be assignable to @x without a cast.
 170  *
 171  * Returns zero on success, or -EFAULT on error.
 172  * On error, the variable @x is set to zero.
 173  */
 174 #define get_user(x,ptr) \
 175         __get_user_check((x), (ptr), sizeof(*(ptr)))
 176
 177 /*
 178  * __put_user: - Write a simple value into user space, with less checking.
 179  * @x:   Value to copy to user space.
 180  * @ptr: Destination address, in user space.
 181  *
 182  * Context: User context only. This function may sleep if pagefaults are
 183  *          enabled.
 184  *
 185  * This macro copies a single simple value from kernel space to user
 186  * space.  It supports simple types like char and int, but not larger
 187  * data types like structures or arrays.
 188  *
 189  * @ptr must have pointer-to-simple-variable type, and @x must be assignable
 190  * to the result of dereferencing @ptr.
 191  *
 192  * Caller must check the pointer with access_ok() before calling this
 193  * function.
 194  *
 195  * Returns zero on success, or -EFAULT on error.
 196  */
 197 #define __put_user(x,ptr) \
 198         __put_user_nocheck((x), (ptr), sizeof(*(ptr)))
 199
 200 /*
 201  * __get_user: - Get a simple variable from user space, with less checking.
 202  * @x:   Variable to store result.
 203  * @ptr: Source address, in user space.
 204  *
 205  * Context: User context only. This function may sleep if pagefaults are
 206  *          enabled.
 207  *
 208  * This macro copies a single simple variable from user space to kernel
 209  * space.  It supports simple types like char and int, but not larger
 210  * data types like structures or arrays.
 211  *
 212  * @ptr must have pointer-to-simple-variable type, and the result of
 213  * dereferencing @ptr must be assignable to @x without a cast.
 214  *
 215  * Caller must check the pointer with access_ok() before calling this
 216  * function.
 217  *
 218  * Returns zero on success, or -EFAULT on error.
 219  * On error, the variable @x is set to zero.
 220  */
 221 #define __get_user(x,ptr) \
 222         __get_user_nocheck((x), (ptr), sizeof(*(ptr)))
 223
 224 struct __large_struct { unsigned long buf[100]; };
 225 #define __m(x) (*(struct __large_struct __user *)(x))
 226
 227 /*
 228  * Yuck.  We need two variants, one for 64bit operation and one
 229  * for 32 bit mode and old iron.
 230  */
 231 #ifndef CONFIG_EVA
 232 #define __get_kernel_common(val, size, ptr) __get_user_common(val, size, ptr)
 233 #else
 234 /*
 235  * Kernel specific functions for EVA. We need to use normal load instructions
 236  * to read data from kernel when operating in EVA mode. We use these macros to
 237  * avoid redefining __get_user_asm for EVA.
 238  */
 239 #undef _loadd
 240 #undef _loadw
 241 #undef _loadh
 242 #undef _loadb
 243 #ifdef CONFIG_32BIT
 244 #define _loadd                  _loadw
 245 #else
 246 #define _loadd(reg, addr)       "ld " reg ", " addr
 247 #endif
 248 #define _loadw(reg, addr)       "lw " reg ", " addr
 249 #define _loadh(reg, addr)       "lh " reg ", " addr
 250 #define _loadb(reg, addr)       "lb " reg ", " addr
 251
 252 #define __get_kernel_common(val, size, ptr)                             \
 253 do {                                                                    \
 254         switch (size) {                                                 \
 255         case 1: __get_data_asm(val, _loadb, ptr); break;                \
 256         case 2: __get_data_asm(val, _loadh, ptr); break;                \
 257         case 4: __get_data_asm(val, _loadw, ptr); break;                \
 258         case 8: __GET_DW(val, _loadd, ptr); break;                      \
 259         default: __get_user_unknown(); break;                           \
 260         }                                                               \
 261 } while (0)
 262 #endif
 263
 264 #ifdef CONFIG_32BIT
 265 #define __GET_DW(val, insn, ptr) __get_data_asm_ll32(val, insn, ptr)
 266 #endif
 267 #ifdef CONFIG_64BIT
 268 #define __GET_DW(val, insn, ptr) __get_data_asm(val, insn, ptr)
 269 #endif
 270
 271 extern void __get_user_unknown(void);
 272
 273 #define __get_user_common(val, size, ptr)                               \
 274 do {                                                                    \
 275         switch (size) {                                                 \
 276         case 1: __get_data_asm(val, user_lb, ptr); break;               \
 277         case 2: __get_data_asm(val, user_lh, ptr); break;               \
 278         case 4: __get_data_asm(val, user_lw, ptr); break;               \
 279         case 8: __GET_DW(val, user_ld, ptr); break;                     \
 280         default: __get_user_unknown(); break;                           \
 281         }                                                               \
 282 } while (0)
 283
 284 #define __get_user_nocheck(x, ptr, size)                                \
 285 ({                                                                      \
 286         int __gu_err;                                                   \
 287                                                                         \
 288         if (eva_kernel_access()) {                                      \
 289                 __get_kernel_common((x), size, ptr);                    \
 290         } else {                                                        \
 291                 __chk_user_ptr(ptr);                                    \
 292                 __get_user_common((x), size, ptr);                      \
 293         }                                                               \
 294         __gu_err;                                                       \
 295 })
 296
 297 #define __get_user_check(x, ptr, size)                                  \
 298 ({                                                                      \
 299         int __gu_err = -EFAULT;                                         \
 300         const __typeof__(*(ptr)) __user * __gu_ptr = (ptr);             \
 301                                                                         \
 302         might_fault();                                                  \
 303         if (likely(access_ok( __gu_ptr, size))) {               \
 304                 if (eva_kernel_access())                                \
 305                         __get_kernel_common((x), size, __gu_ptr);       \
 306                 else                                                    \
 307                         __get_user_common((x), size, __gu_ptr);         \
 308         } else                                                          \
 309                 (x) = 0;                                                \
 310                                                                         \
 311         __gu_err;                                                       \
 312 })
 313
 314 #define __get_data_asm(val, insn, addr)                                 \
 315 {                                                                       \
 316         long __gu_tmp;                                                  \
 317                                                                         \
 318         __asm__ __volatile__(                                           \
 319         "1:     "insn("%1", "%3")"                              \n"     \
 320         "2:                                                     \n"     \
 321         "       .insn                                           \n"     \
 322         "       .section .fixup,\"ax\"                          \n"     \
 323         "3:     li      %0, %4                                  \n"     \
 324         "       move    %1, $0                                  \n"     \
 325         "       j       2b                                      \n"     \
 326         "       .previous                                       \n"     \
 327         "       .section __ex_table,\"a\"                       \n"     \
 328         "       "__UA_ADDR "\t1b, 3b                            \n"     \
 329         "       .previous                                       \n"     \
 330         : "=r" (__gu_err), "=r" (__gu_tmp)                              \
 331         : "0" (0), "o" (__m(addr)), "i" (-EFAULT));                     \
 332                                                                         \
 333         (val) = (__typeof__(*(addr))) __gu_tmp;                         \
 334 }
 335
 336 /*
 337  * Get a long long 64 using 32 bit registers.
 338  */
 339 #define __get_data_asm_ll32(val, insn, addr)                            \
 340 {                                                                       \
 341         union {                                                         \
 342                 unsigned long long      l;                              \
 343                 __typeof__(*(addr))     t;                              \
 344         } __gu_tmp;                                                     \
 345                                                                         \
 346         __asm__ __volatile__(                                           \
 347         "1:     " insn("%1", "(%3)")"                           \n"     \
 348         "2:     " insn("%D1", "4(%3)")"                         \n"     \
 349         "3:                                                     \n"     \
 350         "       .insn                                           \n"     \
 351         "       .section        .fixup,\"ax\"                   \n"     \
 352         "4:     li      %0, %4                                  \n"     \
 353         "       move    %1, $0                                  \n"     \
 354         "       move    %D1, $0                                 \n"     \
 355         "       j       3b                                      \n"     \
 356         "       .previous                                       \n"     \
 357         "       .section        __ex_table,\"a\"                \n"     \
 358         "       " __UA_ADDR "   1b, 4b                          \n"     \
 359         "       " __UA_ADDR "   2b, 4b                          \n"     \
 360         "       .previous                                       \n"     \
 361         : "=r" (__gu_err), "=&r" (__gu_tmp.l)                           \
 362         : "0" (0), "r" (addr), "i" (-EFAULT));                          \
 363                                                                         \
 364         (val) = __gu_tmp.t;                                             \
 365 }
 366
 367 #ifndef CONFIG_EVA
 368 #define __put_kernel_common(ptr, size) __put_user_common(ptr, size)
 369 #else
 370 /*
 371  * Kernel specific functions for EVA. We need to use normal load instructions
 372  * to read data from kernel when operating in EVA mode. We use these macros to
 373  * avoid redefining __get_data_asm for EVA.
 374  */
 375 #undef _stored
 376 #undef _storew
 377 #undef _storeh
 378 #undef _storeb
 379 #ifdef CONFIG_32BIT
 380 #define _stored                 _storew
 381 #else
 382 #define _stored(reg, addr)      "ld " reg ", " addr
 383 #endif
 384
 385 #define _storew(reg, addr)      "sw " reg ", " addr
 386 #define _storeh(reg, addr)      "sh " reg ", " addr
 387 #define _storeb(reg, addr)      "sb " reg ", " addr
 388
 389 #define __put_kernel_common(ptr, size)                                  \
 390 do {                                                                    \
 391         switch (size) {                                                 \
 392         case 1: __put_data_asm(_storeb, ptr); break;                    \
 393         case 2: __put_data_asm(_storeh, ptr); break;                    \
 394         case 4: __put_data_asm(_storew, ptr); break;                    \
 395         case 8: __PUT_DW(_stored, ptr); break;                          \
 396         default: __put_user_unknown(); break;                           \
 397         }                                                               \
 398 } while(0)
 399 #endif
 400
 401 /*
 402  * Yuck.  We need two variants, one for 64bit operation and one
 403  * for 32 bit mode and old iron.
 404  */
 405 #ifdef CONFIG_32BIT
 406 #define __PUT_DW(insn, ptr) __put_data_asm_ll32(insn, ptr)
 407 #endif
 408 #ifdef CONFIG_64BIT
 409 #define __PUT_DW(insn, ptr) __put_data_asm(insn, ptr)
 410 #endif
 411
 412 #define __put_user_common(ptr, size)                                    \
 413 do {                                                                    \
 414         switch (size) {                                                 \
 415         case 1: __put_data_asm(user_sb, ptr); break;                    \
 416         case 2: __put_data_asm(user_sh, ptr); break;                    \
 417         case 4: __put_data_asm(user_sw, ptr); break;                    \
 418         case 8: __PUT_DW(user_sd, ptr); break;                          \
 419         default: __put_user_unknown(); break;                           \
 420         }                                                               \
 421 } while (0)
 422
 423 #define __put_user_nocheck(x, ptr, size)                                \
 424 ({                                                                      \
 425         __typeof__(*(ptr)) __pu_val;                                    \
 426         int __pu_err = 0;                                               \
 427                                                                         \
 428         __pu_val = (x);                                                 \
 429         if (eva_kernel_access()) {                                      \
 430                 __put_kernel_common(ptr, size);                         \
 431         } else {                                                        \
 432                 __chk_user_ptr(ptr);                                    \
 433                 __put_user_common(ptr, size);                           \
 434         }                                                               \
 435         __pu_err;                                                       \
 436 })
 437
 438 #define __put_user_check(x, ptr, size)                                  \
 439 ({                                                                      \
 440         __typeof__(*(ptr)) __user *__pu_addr = (ptr);                   \
 441         __typeof__(*(ptr)) __pu_val = (x);                              \
 442         int __pu_err = -EFAULT;                                         \
 443                                                                         \
 444         might_fault();                                                  \
 445         if (likely(access_ok( __pu_addr, size))) {      \
 446                 if (eva_kernel_access())                                \
 447                         __put_kernel_common(__pu_addr, size);           \
 448                 else                                                    \
 449                         __put_user_common(__pu_addr, size);             \
 450         }                                                               \
 451                                                                         \
 452         __pu_err;                                                       \
 453 })
 454
 455 #define __put_data_asm(insn, ptr)                                       \
 456 {                                                                       \
 457         __asm__ __volatile__(                                           \
 458         "1:     "insn("%z2", "%3")"     # __put_data_asm        \n"     \
 459         "2:                                                     \n"     \
 460         "       .insn                                           \n"     \
 461         "       .section        .fixup,\"ax\"                   \n"     \
 462         "3:     li      %0, %4                                  \n"     \
 463         "       j       2b                                      \n"     \
 464         "       .previous                                       \n"     \
 465         "       .section        __ex_table,\"a\"                \n"     \
 466         "       " __UA_ADDR "   1b, 3b                          \n"     \
 467         "       .previous                                       \n"     \
 468         : "=r" (__pu_err)                                               \
 469         : "0" (0), "Jr" (__pu_val), "o" (__m(ptr)),                     \
 470           "i" (-EFAULT));                                               \
 471 }
 472
 473 #define __put_data_asm_ll32(insn, ptr)                                  \
 474 {                                                                       \
 475         __asm__ __volatile__(                                           \
 476         "1:     "insn("%2", "(%3)")"    # __put_data_asm_ll32   \n"     \
 477         "2:     "insn("%D2", "4(%3)")"                          \n"     \
 478         "3:                                                     \n"     \
 479         "       .insn                                           \n"     \
 480         "       .section        .fixup,\"ax\"                   \n"     \
 481         "4:     li      %0, %4                                  \n"     \
 482         "       j       3b                                      \n"     \
 483         "       .previous                                       \n"     \
 484         "       .section        __ex_table,\"a\"                \n"     \
 485         "       " __UA_ADDR "   1b, 4b                          \n"     \
 486         "       " __UA_ADDR "   2b, 4b                          \n"     \
 487         "       .previous"                                              \
 488         : "=r" (__pu_err)                                               \
 489         : "0" (0), "r" (__pu_val), "r" (ptr),                           \
 490           "i" (-EFAULT));                                               \
 491 }
 492
 493 extern void __put_user_unknown(void);
 494
 495 /*
 496  * We're generating jump to subroutines which will be outside the range of
 497  * jump instructions
 498  */
 499 #ifdef MODULE
 500 #define __MODULE_JAL(destination)                                       \
 501         ".set\tnoat\n\t"                                                \
 502         __UA_LA "\t$1, " #destination "\n\t"                            \
 503         "jalr\t$1\n\t"                                                  \
 504         ".set\tat\n\t"
 505 #else
 506 #define __MODULE_JAL(destination)                                       \
 507         "jal\t" #destination "\n\t"
 508 #endif
 509
 510 #if defined(CONFIG_CPU_DADDI_WORKAROUNDS) || (defined(CONFIG_EVA) &&    \
 511                                               defined(CONFIG_CPU_HAS_PREFETCH))
 512 #define DADDI_SCRATCH "$3"
 513 #else
 514 #define DADDI_SCRATCH "$0"
 515 #endif
 516
 517 extern size_t __copy_user(void *__to, const void *__from, size_t __n);
 518
 519 #define __invoke_copy_from(func, to, from, n)                           \
 520 ({                                                                      \
 521         register void *__cu_to_r __asm__("$4");                         \
 522         register const void __user *__cu_from_r __asm__("$5");          \
 523         register long __cu_len_r __asm__("$6");                         \
 524                                                                         \
 525         __cu_to_r = (to);                                               \
 526         __cu_from_r = (from);                                           \
 527         __cu_len_r = (n);                                               \
 528         __asm__ __volatile__(                                           \
 529         ".set\tnoreorder\n\t"                                           \
 530         __MODULE_JAL(func)                                              \
 531         ".set\tnoat\n\t"                                                \
 532         __UA_ADDU "\t$1, %1, %2\n\t"                                    \
 533         ".set\tat\n\t"                                                  \
 534         ".set\treorder"                                                 \
 535         : "+r" (__cu_to_r), "+r" (__cu_from_r), "+r" (__cu_len_r)       \
 536         :                                                               \
 537         : "$8", "$9", "$10", "$11", "$12", "$14", "$15", "$24", "$31",  \
 538           DADDI_SCRATCH, "memory");                                     \
 539         __cu_len_r;                                                     \
 540 })
 541
 542 #define __invoke_copy_to(func, to, from, n)                             \
 543 ({                                                                      \
 544         register void __user *__cu_to_r __asm__("$4");                  \
 545         register const void *__cu_from_r __asm__("$5");                 \
 546         register long __cu_len_r __asm__("$6");                         \
 547                                                                         \
 548         __cu_to_r = (to);                                               \
 549         __cu_from_r = (from);                                           \
 550         __cu_len_r = (n);                                               \
 551         __asm__ __volatile__(                                           \
 552         __MODULE_JAL(func)                                              \
 553         : "+r" (__cu_to_r), "+r" (__cu_from_r), "+r" (__cu_len_r)       \
 554         :                                                               \
 555         : "$8", "$9", "$10", "$11", "$12", "$14", "$15", "$24", "$31",  \
 556           DADDI_SCRATCH, "memory");                                     \
 557         __cu_len_r;                                                     \
 558 })
 559
 560 #define __invoke_copy_from_kernel(to, from, n)                          \
 561         __invoke_copy_from(__copy_user, to, from, n)
 562
 563 #define __invoke_copy_to_kernel(to, from, n)                            \
 564         __invoke_copy_to(__copy_user, to, from, n)
 565
 566 #define ___invoke_copy_in_kernel(to, from, n)                           \
 567         __invoke_copy_from(__copy_user, to, from, n)
 568
 569 #ifndef CONFIG_EVA
 570 #define __invoke_copy_from_user(to, from, n)                            \
 571         __invoke_copy_from(__copy_user, to, from, n)
 572
 573 #define __invoke_copy_to_user(to, from, n)                              \
 574         __invoke_copy_to(__copy_user, to, from, n)
 575
 576 #define ___invoke_copy_in_user(to, from, n)                             \
 577         __invoke_copy_from(__copy_user, to, from, n)
 578
 579 #else
 580
 581 /* EVA specific functions */
 582
 583 extern size_t __copy_from_user_eva(void *__to, const void *__from,
 584                                    size_t __n);
 585 extern size_t __copy_to_user_eva(void *__to, const void *__from,
 586                                  size_t __n);
 587 extern size_t __copy_in_user_eva(void *__to, const void *__from, size_t __n);
 588
 589 /*
 590  * Source or destination address is in userland. We need to go through
 591  * the TLB
 592  */
 593 #define __invoke_copy_from_user(to, from, n)                            \
 594         __invoke_copy_from(__copy_from_user_eva, to, from, n)
 595
 596 #define __invoke_copy_to_user(to, from, n)                              \
 597         __invoke_copy_to(__copy_to_user_eva, to, from, n)
 598
 599 #define ___invoke_copy_in_user(to, from, n)                             \
 600         __invoke_copy_from(__copy_in_user_eva, to, from, n)
 601
 602 #endif /* CONFIG_EVA */
 603
 604 static inline unsigned long
 605 raw_copy_to_user(void __user *to, const void *from, unsigned long n)
 606 {
 607         if (eva_kernel_access())
 608                 return __invoke_copy_to_kernel(to, from, n);
 609         else
 610                 return __invoke_copy_to_user(to, from, n);
 611 }
 612
 613 static inline unsigned long
 614 raw_copy_from_user(void *to, const void __user *from, unsigned long n)
 615 {
 616         if (eva_kernel_access())
 617                 return __invoke_copy_from_kernel(to, from, n);
 618         else
 619                 return __invoke_copy_from_user(to, from, n);
 620 }
 621
 622 #define INLINE_COPY_FROM_USER
 623 #define INLINE_COPY_TO_USER
 624
 625 static inline unsigned long
 626 raw_copy_in_user(void __user*to, const void __user *from, unsigned long n)
 627 {
 628         if (eva_kernel_access())
 629                 return ___invoke_copy_in_kernel(to, from, n);
 630         else
 631                 return ___invoke_copy_in_user(to, from, n);
 632 }
 633
 634 extern __kernel_size_t __bzero_kernel(void __user *addr, __kernel_size_t size);
 635 extern __kernel_size_t __bzero(void __user *addr, __kernel_size_t size);
 636
 637 /*
 638  * __clear_user: - Zero a block of memory in user space, with less checking.
 639  * @to:   Destination address, in user space.
 640  * @n:    Number of bytes to zero.
 641  *
 642  * Zero a block of memory in user space.  Caller must check
 643  * the specified block with access_ok() before calling this function.
 644  *
 645  * Returns number of bytes that could not be cleared.
 646  * On success, this will be zero.
 647  */
 648 static inline __kernel_size_t
 649 __clear_user(void __user *addr, __kernel_size_t size)
 650 {
 651         __kernel_size_t res;
 652
 653 #ifdef CONFIG_CPU_MICROMIPS
 654 /* micromips memset / bzero also clobbers t7 & t8 */
 655 #define bzero_clobbers "$4", "$5", "$6", __UA_t0, __UA_t1, "$15", "$24", "$31"
 656 #else
 657 #define bzero_clobbers "$4", "$5", "$6", __UA_t0, __UA_t1, "$31"
 658 #endif /* CONFIG_CPU_MICROMIPS */
 659
 660         if (eva_kernel_access()) {
 661                 __asm__ __volatile__(
 662                         "move\t$4, %1\n\t"
 663                         "move\t$5, $0\n\t"
 664                         "move\t$6, %2\n\t"
 665                         __MODULE_JAL(__bzero_kernel)
 666                         "move\t%0, $6"
 667                         : "=r" (res)
 668                         : "r" (addr), "r" (size)
 669                         : bzero_clobbers);
 670         } else {
 671                 might_fault();
 672                 __asm__ __volatile__(
 673                         "move\t$4, %1\n\t"
 674                         "move\t$5, $0\n\t"
 675                         "move\t$6, %2\n\t"
 676                         __MODULE_JAL(__bzero)
 677                         "move\t%0, $6"
 678                         : "=r" (res)
 679                         : "r" (addr), "r" (size)
 680                         : bzero_clobbers);
 681         }
 682
 683         return res;
 684 }
 685
 686 #define clear_user(addr,n)                                              \
 687 ({                                                                      \
 688         void __user * __cl_addr = (addr);                               \
 689         unsigned long __cl_size = (n);                                  \
 690         if (__cl_size && access_ok(__cl_addr, __cl_size))               \
 691                 __cl_size = __clear_user(__cl_addr, __cl_size);         \
 692         __cl_size;                                                      \
 693 })
 694
 695 extern long __strncpy_from_kernel_asm(char *__to, const char __user *__from, long __len);
 696 extern long __strncpy_from_user_asm(char *__to, const char __user *__from, long __len);
 697
 698 /*
 699  * strncpy_from_user: - Copy a NUL terminated string from userspace.
 700  * @dst:   Destination address, in kernel space.  This buffer must be at
 701  *         least @count bytes long.
 702  * @src:   Source address, in user space.
 703  * @count: Maximum number of bytes to copy, including the trailing NUL.
 704  *
 705  * Copies a NUL-terminated string from userspace to kernel space.
 706  *
 707  * On success, returns the length of the string (not including the trailing
 708  * NUL).
 709  *
 710  * If access to userspace fails, returns -EFAULT (some data may have been
 711  * copied).
 712  *
 713  * If @count is smaller than the length of the string, copies @count bytes
 714  * and returns @count.
 715  */
 716 static inline long
 717 strncpy_from_user(char *__to, const char __user *__from, long __len)
 718 {
 719         long res;
 720
 721         if (eva_kernel_access()) {
 722                 __asm__ __volatile__(
 723                         "move\t$4, %1\n\t"
 724                         "move\t$5, %2\n\t"
 725                         "move\t$6, %3\n\t"
 726                         __MODULE_JAL(__strncpy_from_kernel_asm)
 727                         "move\t%0, $2"
 728                         : "=r" (res)
 729                         : "r" (__to), "r" (__from), "r" (__len)
 730                         : "$2", "$3", "$4", "$5", "$6", __UA_t0, "$31", "memory");
 731         } else {
 732                 might_fault();
 733                 __asm__ __volatile__(
 734                         "move\t$4, %1\n\t"
 735                         "move\t$5, %2\n\t"
 736                         "move\t$6, %3\n\t"
 737                         __MODULE_JAL(__strncpy_from_user_asm)
 738                         "move\t%0, $2"
 739                         : "=r" (res)
 740                         : "r" (__to), "r" (__from), "r" (__len)
 741                         : "$2", "$3", "$4", "$5", "$6", __UA_t0, "$31", "memory");
 742         }
 743
 744         return res;
 745 }
 746
 747 extern long __strnlen_kernel_asm(const char __user *s, long n);
 748 extern long __strnlen_user_asm(const char __user *s, long n);
 749
 750 /*
 751  * strnlen_user: - Get the size of a string in user space.
 752  * @str: The string to measure.
 753  *
 754  * Context: User context only. This function may sleep if pagefaults are
 755  *          enabled.
 756  *
 757  * Get the size of a NUL-terminated string in user space.
 758  *
 759  * Returns the size of the string INCLUDING the terminating NUL.
 760  * On exception, returns 0.
 761  * If the string is too long, returns a value greater than @n.
 762  */
 763 static inline long strnlen_user(const char __user *s, long n)
 764 {
 765         long res;
 766
 767         might_fault();
 768         if (eva_kernel_access()) {
 769                 __asm__ __volatile__(
 770                         "move\t$4, %1\n\t"
 771                         "move\t$5, %2\n\t"
 772                         __MODULE_JAL(__strnlen_kernel_asm)
 773                         "move\t%0, $2"
 774                         : "=r" (res)
 775                         : "r" (s), "r" (n)
 776                         : "$2", "$4", "$5", __UA_t0, "$31");
 777         } else {
 778                 __asm__ __volatile__(
 779                         "move\t$4, %1\n\t"
 780                         "move\t$5, %2\n\t"
 781                         __MODULE_JAL(__strnlen_user_asm)
 782                         "move\t%0, $2"
 783                         : "=r" (res)
 784                         : "r" (s), "r" (n)
 785                         : "$2", "$4", "$5", __UA_t0, "$31");
 786         }
 787
 788         return res;
 789 }
 790
 791 #endif /* _ASM_UACCESS_H */