arch/mips/kernel/unaligned.c

   1 /*
   2  * Handle unaligned accesses by emulation.
   3  *
   4  * This file is subject to the terms and conditions of the GNU General Public
   5  * License.  See the file "COPYING" in the main directory of this archive
   6  * for more details.
   7  *
   8  * Copyright (C) 1996, 1998, 1999, 2002 by Ralf Baechle
   9  * Copyright (C) 1999 Silicon Graphics, Inc.
  10  * Copyright (C) 2014 Imagination Technologies Ltd.
  11  *
  12  * This file contains exception handler for address error exception with the
  13  * special capability to execute faulting instructions in software.  The
  14  * handler does not try to handle the case when the program counter points
  15  * to an address not aligned to a word boundary.
  16  *
  17  * Putting data to unaligned addresses is a bad practice even on Intel where
  18  * only the performance is affected.  Much worse is that such code is non-
  19  * portable.  Due to several programs that die on MIPS due to alignment
  20  * problems I decided to implement this handler anyway though I originally
  21  * didn't intend to do this at all for user code.
  22  *
  23  * For now I enable fixing of address errors by default to make life easier.
  24  * I however intend to disable this somewhen in the future when the alignment
  25  * problems with user programs have been fixed.  For programmers this is the
  26  * right way to go.
  27  *
  28  * Fixing address errors is a per process option.  The option is inherited
  29  * across fork(2) and execve(2) calls.  If you really want to use the
  30  * option in your user programs - I discourage the use of the software
  31  * emulation strongly - use the following code in your userland stuff:
  32  *
  33  * #include <sys/sysmips.h>
  34  *
  35  * ...
  36  * sysmips(MIPS_FIXADE, x);
  37  * ...
  38  *
  39  * The argument x is 0 for disabling software emulation, enabled otherwise.
  40  *
  41  * Below a little program to play around with this feature.
  42  *
  43  * #include <stdio.h>
  44  * #include <sys/sysmips.h>
  45  *
  46  * struct foo {
  47  *         unsigned char bar[8];
  48  * };
  49  *
  50  * main(int argc, char *argv[])
  51  * {
  52  *         struct foo x = {0, 1, 2, 3, 4, 5, 6, 7};
  53  *         unsigned int *p = (unsigned int *) (x.bar + 3);
  54  *         int i;
  55  *
  56  *         if (argc > 1)
  57  *                 sysmips(MIPS_FIXADE, atoi(argv[1]));
  58  *
  59  *         printf("*p = %08lx\n", *p);
  60  *
  61  *         *p = 0xdeadface;
  62  *
  63  *         for(i = 0; i <= 7; i++)
  64  *         printf("%02x ", x.bar[i]);
  65  *         printf("\n");
  66  * }
  67  *
  68  * Coprocessor loads are not supported; I think this case is unimportant
  69  * in the practice.
  70  *
  71  * TODO: Handle ndc (attempted store to doubleword in uncached memory)
  72  *       exception for the R6000.
  73  *       A store crossing a page boundary might be executed only partially.
  74  *       Undo the partial store in this case.
  75  */
  76 #include <linux/context_tracking.h>
  77 #include <linux/mm.h>
  78 #include <linux/signal.h>
  79 #include <linux/smp.h>
  80 #include <linux/sched.h>
  81 #include <linux/debugfs.h>
  82 #include <linux/perf_event.h>
  83
  84 #include <asm/asm.h>
  85 #include <asm/branch.h>
  86 #include <asm/byteorder.h>
  87 #include <asm/cop2.h>
  88 #include <asm/debug.h>
  89 #include <asm/fpu.h>
  90 #include <asm/fpu_emulator.h>
  91 #include <asm/inst.h>
  92 #include <asm/mmu_context.h>
  93 #include <linux/uaccess.h>
  94
  95 #define STR(x)  __STR(x)
  96 #define __STR(x)  #x
  97
  98 enum {
  99         UNALIGNED_ACTION_QUIET,
 100         UNALIGNED_ACTION_SIGNAL,
 101         UNALIGNED_ACTION_SHOW,
 102 };
 103 #ifdef CONFIG_DEBUG_FS
 104 static u32 unaligned_instructions;
 105 static u32 unaligned_action;
 106 #else
 107 #define unaligned_action UNALIGNED_ACTION_QUIET
 108 #endif
 109 extern void show_registers(struct pt_regs *regs);
 110
 111 #ifdef __BIG_ENDIAN
 112 #define     _LoadHW(addr, value, res, type)  \
 113 do {                                                        \
 114                 __asm__ __volatile__ (".set\tnoat\n"        \
 115                         "1:\t"type##_lb("%0", "0(%2)")"\n"  \
 116                         "2:\t"type##_lbu("$1", "1(%2)")"\n\t"\
 117                         "sll\t%0, 0x8\n\t"                  \
 118                         "or\t%0, $1\n\t"                    \
 119                         "li\t%1, 0\n"                       \
 120                         "3:\t.set\tat\n\t"                  \
 121                         ".insn\n\t"                         \
 122                         ".section\t.fixup,\"ax\"\n\t"       \
 123                         "4:\tli\t%1, %3\n\t"                \
 124                         "j\t3b\n\t"                         \
 125                         ".previous\n\t"                     \
 126                         ".section\t__ex_table,\"a\"\n\t"    \
 127                         STR(PTR)"\t1b, 4b\n\t"              \
 128                         STR(PTR)"\t2b, 4b\n\t"              \
 129                         ".previous"                         \
 130                         : "=&r" (value), "=r" (res)         \
 131                         : "r" (addr), "i" (-EFAULT));       \
 132 } while(0)
 133
 134 #ifdef CONFIG_CPU_HAS_LOAD_STORE_LR
 135 #define     _LoadW(addr, value, res, type)   \
 136 do {                                                        \
 137                 __asm__ __volatile__ (                      \
 138                         "1:\t"type##_lwl("%0", "(%2)")"\n"   \
 139                         "2:\t"type##_lwr("%0", "3(%2)")"\n\t"\
 140                         "li\t%1, 0\n"                       \
 141                         "3:\n\t"                            \
 142                         ".insn\n\t"                         \
 143                         ".section\t.fixup,\"ax\"\n\t"       \
 144                         "4:\tli\t%1, %3\n\t"                \
 145                         "j\t3b\n\t"                         \
 146                         ".previous\n\t"                     \
 147                         ".section\t__ex_table,\"a\"\n\t"    \
 148                         STR(PTR)"\t1b, 4b\n\t"              \
 149                         STR(PTR)"\t2b, 4b\n\t"              \
 150                         ".previous"                         \
 151                         : "=&r" (value), "=r" (res)         \
 152                         : "r" (addr), "i" (-EFAULT));       \
 153 } while(0)
 154
 155 #else /* !CONFIG_CPU_HAS_LOAD_STORE_LR */
 156 /* For CPUs without lwl instruction */
 157 #define     _LoadW(addr, value, res, type) \
 158 do {                                                        \
 159                 __asm__ __volatile__ (                      \
 160                         ".set\tpush\n"                      \
 161                         ".set\tnoat\n\t"                    \
 162                         "1:"type##_lb("%0", "0(%2)")"\n\t"  \
 163                         "2:"type##_lbu("$1", "1(%2)")"\n\t" \
 164                         "sll\t%0, 0x8\n\t"                  \
 165                         "or\t%0, $1\n\t"                    \
 166                         "3:"type##_lbu("$1", "2(%2)")"\n\t" \
 167                         "sll\t%0, 0x8\n\t"                  \
 168                         "or\t%0, $1\n\t"                    \
 169                         "4:"type##_lbu("$1", "3(%2)")"\n\t" \
 170                         "sll\t%0, 0x8\n\t"                  \
 171                         "or\t%0, $1\n\t"                    \
 172                         "li\t%1, 0\n"                       \
 173                         ".set\tpop\n"                       \
 174                         "10:\n\t"                           \
 175                         ".insn\n\t"                         \
 176                         ".section\t.fixup,\"ax\"\n\t"       \
 177                         "11:\tli\t%1, %3\n\t"               \
 178                         "j\t10b\n\t"                        \
 179                         ".previous\n\t"                     \
 180                         ".section\t__ex_table,\"a\"\n\t"    \
 181                         STR(PTR)"\t1b, 11b\n\t"             \
 182                         STR(PTR)"\t2b, 11b\n\t"             \
 183                         STR(PTR)"\t3b, 11b\n\t"             \
 184                         STR(PTR)"\t4b, 11b\n\t"             \
 185                         ".previous"                         \
 186                         : "=&r" (value), "=r" (res)         \
 187                         : "r" (addr), "i" (-EFAULT));       \
 188 } while(0)
 189
 190 #endif /* !CONFIG_CPU_HAS_LOAD_STORE_LR */
 191
 192 #define     _LoadHWU(addr, value, res, type) \
 193 do {                                                        \
 194                 __asm__ __volatile__ (                      \
 195                         ".set\tnoat\n"                      \
 196                         "1:\t"type##_lbu("%0", "0(%2)")"\n" \
 197                         "2:\t"type##_lbu("$1", "1(%2)")"\n\t"\
 198                         "sll\t%0, 0x8\n\t"                  \
 199                         "or\t%0, $1\n\t"                    \
 200                         "li\t%1, 0\n"                       \
 201                         "3:\n\t"                            \
 202                         ".insn\n\t"                         \
 203                         ".set\tat\n\t"                      \
 204                         ".section\t.fixup,\"ax\"\n\t"       \
 205                         "4:\tli\t%1, %3\n\t"                \
 206                         "j\t3b\n\t"                         \
 207                         ".previous\n\t"                     \
 208                         ".section\t__ex_table,\"a\"\n\t"    \
 209                         STR(PTR)"\t1b, 4b\n\t"              \
 210                         STR(PTR)"\t2b, 4b\n\t"              \
 211                         ".previous"                         \
 212                         : "=&r" (value), "=r" (res)         \
 213                         : "r" (addr), "i" (-EFAULT));       \
 214 } while(0)
 215
 216 #ifdef CONFIG_CPU_HAS_LOAD_STORE_LR
 217 #define     _LoadWU(addr, value, res, type)  \
 218 do {                                                        \
 219                 __asm__ __volatile__ (                      \
 220                         "1:\t"type##_lwl("%0", "(%2)")"\n"  \
 221                         "2:\t"type##_lwr("%0", "3(%2)")"\n\t"\
 222                         "dsll\t%0, %0, 32\n\t"              \
 223                         "dsrl\t%0, %0, 32\n\t"              \
 224                         "li\t%1, 0\n"                       \
 225                         "3:\n\t"                            \
 226                         ".insn\n\t"                         \
 227                         "\t.section\t.fixup,\"ax\"\n\t"     \
 228                         "4:\tli\t%1, %3\n\t"                \
 229                         "j\t3b\n\t"                         \
 230                         ".previous\n\t"                     \
 231                         ".section\t__ex_table,\"a\"\n\t"    \
 232                         STR(PTR)"\t1b, 4b\n\t"              \
 233                         STR(PTR)"\t2b, 4b\n\t"              \
 234                         ".previous"                         \
 235                         : "=&r" (value), "=r" (res)         \
 236                         : "r" (addr), "i" (-EFAULT));       \
 237 } while(0)
 238
 239 #define     _LoadDW(addr, value, res)  \
 240 do {                                                        \
 241                 __asm__ __volatile__ (                      \
 242                         "1:\tldl\t%0, (%2)\n"               \
 243                         "2:\tldr\t%0, 7(%2)\n\t"            \
 244                         "li\t%1, 0\n"                       \
 245                         "3:\n\t"                            \
 246                         ".insn\n\t"                         \
 247                         "\t.section\t.fixup,\"ax\"\n\t"     \
 248                         "4:\tli\t%1, %3\n\t"                \
 249                         "j\t3b\n\t"                         \
 250                         ".previous\n\t"                     \
 251                         ".section\t__ex_table,\"a\"\n\t"    \
 252                         STR(PTR)"\t1b, 4b\n\t"              \
 253                         STR(PTR)"\t2b, 4b\n\t"              \
 254                         ".previous"                         \
 255                         : "=&r" (value), "=r" (res)         \
 256                         : "r" (addr), "i" (-EFAULT));       \
 257 } while(0)
 258
 259 #else /* !CONFIG_CPU_HAS_LOAD_STORE_LR */
 260 /* For CPUs without lwl and ldl instructions */
 261 #define     _LoadWU(addr, value, res, type) \
 262 do {                                                        \
 263                 __asm__ __volatile__ (                      \
 264                         ".set\tpush\n\t"                    \
 265                         ".set\tnoat\n\t"                    \
 266                         "1:"type##_lbu("%0", "0(%2)")"\n\t" \
 267                         "2:"type##_lbu("$1", "1(%2)")"\n\t" \
 268                         "sll\t%0, 0x8\n\t"                  \
 269                         "or\t%0, $1\n\t"                    \
 270                         "3:"type##_lbu("$1", "2(%2)")"\n\t" \
 271                         "sll\t%0, 0x8\n\t"                  \
 272                         "or\t%0, $1\n\t"                    \
 273                         "4:"type##_lbu("$1", "3(%2)")"\n\t" \
 274                         "sll\t%0, 0x8\n\t"                  \
 275                         "or\t%0, $1\n\t"                    \
 276                         "li\t%1, 0\n"                       \
 277                         ".set\tpop\n"                       \
 278                         "10:\n\t"                           \
 279                         ".insn\n\t"                         \
 280                         ".section\t.fixup,\"ax\"\n\t"       \
 281                         "11:\tli\t%1, %3\n\t"               \
 282                         "j\t10b\n\t"                        \
 283                         ".previous\n\t"                     \
 284                         ".section\t__ex_table,\"a\"\n\t"    \
 285                         STR(PTR)"\t1b, 11b\n\t"             \
 286                         STR(PTR)"\t2b, 11b\n\t"             \
 287                         STR(PTR)"\t3b, 11b\n\t"             \
 288                         STR(PTR)"\t4b, 11b\n\t"             \
 289                         ".previous"                         \
 290                         : "=&r" (value), "=r" (res)         \
 291                         : "r" (addr), "i" (-EFAULT));       \
 292 } while(0)
 293
 294 #define     _LoadDW(addr, value, res)  \
 295 do {                                                        \
 296                 __asm__ __volatile__ (                      \
 297                         ".set\tpush\n\t"                    \
 298                         ".set\tnoat\n\t"                    \
 299                         "1:lb\t%0, 0(%2)\n\t"               \
 300                         "2:lbu\t $1, 1(%2)\n\t"             \
 301                         "dsll\t%0, 0x8\n\t"                 \
 302                         "or\t%0, $1\n\t"                    \
 303                         "3:lbu\t$1, 2(%2)\n\t"              \
 304                         "dsll\t%0, 0x8\n\t"                 \
 305                         "or\t%0, $1\n\t"                    \
 306                         "4:lbu\t$1, 3(%2)\n\t"              \
 307                         "dsll\t%0, 0x8\n\t"                 \
 308                         "or\t%0, $1\n\t"                    \
 309                         "5:lbu\t$1, 4(%2)\n\t"              \
 310                         "dsll\t%0, 0x8\n\t"                 \
 311                         "or\t%0, $1\n\t"                    \
 312                         "6:lbu\t$1, 5(%2)\n\t"              \
 313                         "dsll\t%0, 0x8\n\t"                 \
 314                         "or\t%0, $1\n\t"                    \
 315                         "7:lbu\t$1, 6(%2)\n\t"              \
 316                         "dsll\t%0, 0x8\n\t"                 \
 317                         "or\t%0, $1\n\t"                    \
 318                         "8:lbu\t$1, 7(%2)\n\t"              \
 319                         "dsll\t%0, 0x8\n\t"                 \
 320                         "or\t%0, $1\n\t"                    \
 321                         "li\t%1, 0\n"                       \
 322                         ".set\tpop\n\t"                     \
 323                         "10:\n\t"                           \
 324                         ".insn\n\t"                         \
 325                         ".section\t.fixup,\"ax\"\n\t"       \
 326                         "11:\tli\t%1, %3\n\t"               \
 327                         "j\t10b\n\t"                        \
 328                         ".previous\n\t"                     \
 329                         ".section\t__ex_table,\"a\"\n\t"    \
 330                         STR(PTR)"\t1b, 11b\n\t"             \
 331                         STR(PTR)"\t2b, 11b\n\t"             \
 332                         STR(PTR)"\t3b, 11b\n\t"             \
 333                         STR(PTR)"\t4b, 11b\n\t"             \
 334                         STR(PTR)"\t5b, 11b\n\t"             \
 335                         STR(PTR)"\t6b, 11b\n\t"             \
 336                         STR(PTR)"\t7b, 11b\n\t"             \
 337                         STR(PTR)"\t8b, 11b\n\t"             \
 338                         ".previous"                         \
 339                         : "=&r" (value), "=r" (res)         \
 340                         : "r" (addr), "i" (-EFAULT));       \
 341 } while(0)
 342
 343 #endif /* !CONFIG_CPU_HAS_LOAD_STORE_LR */
 344
 345
 346 #define     _StoreHW(addr, value, res, type) \
 347 do {                                                        \
 348                 __asm__ __volatile__ (                      \
 349                         ".set\tnoat\n"                      \
 350                         "1:\t"type##_sb("%1", "1(%2)")"\n"  \
 351                         "srl\t$1, %1, 0x8\n"                \
 352                         "2:\t"type##_sb("$1", "0(%2)")"\n"  \
 353                         ".set\tat\n\t"                      \
 354                         "li\t%0, 0\n"                       \
 355                         "3:\n\t"                            \
 356                         ".insn\n\t"                         \
 357                         ".section\t.fixup,\"ax\"\n\t"       \
 358                         "4:\tli\t%0, %3\n\t"                \
 359                         "j\t3b\n\t"                         \
 360                         ".previous\n\t"                     \
 361                         ".section\t__ex_table,\"a\"\n\t"    \
 362                         STR(PTR)"\t1b, 4b\n\t"              \
 363                         STR(PTR)"\t2b, 4b\n\t"              \
 364                         ".previous"                         \
 365                         : "=r" (res)                        \
 366                         : "r" (value), "r" (addr), "i" (-EFAULT));\
 367 } while(0)
 368
 369 #ifdef CONFIG_CPU_HAS_LOAD_STORE_LR
 370 #define     _StoreW(addr, value, res, type)  \
 371 do {                                                        \
 372                 __asm__ __volatile__ (                      \
 373                         "1:\t"type##_swl("%1", "(%2)")"\n"  \
 374                         "2:\t"type##_swr("%1", "3(%2)")"\n\t"\
 375                         "li\t%0, 0\n"                       \
 376                         "3:\n\t"                            \
 377                         ".insn\n\t"                         \
 378                         ".section\t.fixup,\"ax\"\n\t"       \
 379                         "4:\tli\t%0, %3\n\t"                \
 380                         "j\t3b\n\t"                         \
 381                         ".previous\n\t"                     \
 382                         ".section\t__ex_table,\"a\"\n\t"    \
 383                         STR(PTR)"\t1b, 4b\n\t"              \
 384                         STR(PTR)"\t2b, 4b\n\t"              \
 385                         ".previous"                         \
 386                 : "=r" (res)                                \
 387                 : "r" (value), "r" (addr), "i" (-EFAULT));  \
 388 } while(0)
 389
 390 #define     _StoreDW(addr, value, res) \
 391 do {                                                        \
 392                 __asm__ __volatile__ (                      \
 393                         "1:\tsdl\t%1,(%2)\n"                \
 394                         "2:\tsdr\t%1, 7(%2)\n\t"            \
 395                         "li\t%0, 0\n"                       \
 396                         "3:\n\t"                            \
 397                         ".insn\n\t"                         \
 398                         ".section\t.fixup,\"ax\"\n\t"       \
 399                         "4:\tli\t%0, %3\n\t"                \
 400                         "j\t3b\n\t"                         \
 401                         ".previous\n\t"                     \
 402                         ".section\t__ex_table,\"a\"\n\t"    \
 403                         STR(PTR)"\t1b, 4b\n\t"              \
 404                         STR(PTR)"\t2b, 4b\n\t"              \
 405                         ".previous"                         \
 406                 : "=r" (res)                                \
 407                 : "r" (value), "r" (addr), "i" (-EFAULT));  \
 408 } while(0)
 409
 410 #else /* !CONFIG_CPU_HAS_LOAD_STORE_LR */
 411 #define     _StoreW(addr, value, res, type)  \
 412 do {                                                        \
 413                 __asm__ __volatile__ (                      \
 414                         ".set\tpush\n\t"                    \
 415                         ".set\tnoat\n\t"                    \
 416                         "1:"type##_sb("%1", "3(%2)")"\n\t"  \
 417                         "srl\t$1, %1, 0x8\n\t"              \
 418                         "2:"type##_sb("$1", "2(%2)")"\n\t"  \
 419                         "srl\t$1, $1,  0x8\n\t"             \
 420                         "3:"type##_sb("$1", "1(%2)")"\n\t"  \
 421                         "srl\t$1, $1, 0x8\n\t"              \
 422                         "4:"type##_sb("$1", "0(%2)")"\n\t"  \
 423                         ".set\tpop\n\t"                     \
 424                         "li\t%0, 0\n"                       \
 425                         "10:\n\t"                           \
 426                         ".insn\n\t"                         \
 427                         ".section\t.fixup,\"ax\"\n\t"       \
 428                         "11:\tli\t%0, %3\n\t"               \
 429                         "j\t10b\n\t"                        \
 430                         ".previous\n\t"                     \
 431                         ".section\t__ex_table,\"a\"\n\t"    \
 432                         STR(PTR)"\t1b, 11b\n\t"             \
 433                         STR(PTR)"\t2b, 11b\n\t"             \
 434                         STR(PTR)"\t3b, 11b\n\t"             \
 435                         STR(PTR)"\t4b, 11b\n\t"             \
 436                         ".previous"                         \
 437                 : "=&r" (res)                               \
 438                 : "r" (value), "r" (addr), "i" (-EFAULT)    \
 439                 : "memory");                                \
 440 } while(0)
 441
 442 #define     _StoreDW(addr, value, res) \
 443 do {                                                        \
 444                 __asm__ __volatile__ (                      \
 445                         ".set\tpush\n\t"                    \
 446                         ".set\tnoat\n\t"                    \
 447                         "1:sb\t%1, 7(%2)\n\t"               \
 448                         "dsrl\t$1, %1, 0x8\n\t"             \
 449                         "2:sb\t$1, 6(%2)\n\t"               \
 450                         "dsrl\t$1, $1, 0x8\n\t"             \
 451                         "3:sb\t$1, 5(%2)\n\t"               \
 452                         "dsrl\t$1, $1, 0x8\n\t"             \
 453                         "4:sb\t$1, 4(%2)\n\t"               \
 454                         "dsrl\t$1, $1, 0x8\n\t"             \
 455                         "5:sb\t$1, 3(%2)\n\t"               \
 456                         "dsrl\t$1, $1, 0x8\n\t"             \
 457                         "6:sb\t$1, 2(%2)\n\t"               \
 458                         "dsrl\t$1, $1, 0x8\n\t"             \
 459                         "7:sb\t$1, 1(%2)\n\t"               \
 460                         "dsrl\t$1, $1, 0x8\n\t"             \
 461                         "8:sb\t$1, 0(%2)\n\t"               \
 462                         "dsrl\t$1, $1, 0x8\n\t"             \
 463                         ".set\tpop\n\t"                     \
 464                         "li\t%0, 0\n"                       \
 465                         "10:\n\t"                           \
 466                         ".insn\n\t"                         \
 467                         ".section\t.fixup,\"ax\"\n\t"       \
 468                         "11:\tli\t%0, %3\n\t"               \
 469                         "j\t10b\n\t"                        \
 470                         ".previous\n\t"                     \
 471                         ".section\t__ex_table,\"a\"\n\t"    \
 472                         STR(PTR)"\t1b, 11b\n\t"             \
 473                         STR(PTR)"\t2b, 11b\n\t"             \
 474                         STR(PTR)"\t3b, 11b\n\t"             \
 475                         STR(PTR)"\t4b, 11b\n\t"             \
 476                         STR(PTR)"\t5b, 11b\n\t"             \
 477                         STR(PTR)"\t6b, 11b\n\t"             \
 478                         STR(PTR)"\t7b, 11b\n\t"             \
 479                         STR(PTR)"\t8b, 11b\n\t"             \
 480                         ".previous"                         \
 481                 : "=&r" (res)                               \
 482                 : "r" (value), "r" (addr), "i" (-EFAULT)    \
 483                 : "memory");                                \
 484 } while(0)
 485
 486 #endif /* !CONFIG_CPU_HAS_LOAD_STORE_LR */
 487
 488 #else /* __BIG_ENDIAN */
 489
 490 #define     _LoadHW(addr, value, res, type)  \
 491 do {                                                        \
 492                 __asm__ __volatile__ (".set\tnoat\n"        \
 493                         "1:\t"type##_lb("%0", "1(%2)")"\n"  \
 494                         "2:\t"type##_lbu("$1", "0(%2)")"\n\t"\
 495                         "sll\t%0, 0x8\n\t"                  \
 496                         "or\t%0, $1\n\t"                    \
 497                         "li\t%1, 0\n"                       \
 498                         "3:\t.set\tat\n\t"                  \
 499                         ".insn\n\t"                         \
 500                         ".section\t.fixup,\"ax\"\n\t"       \
 501                         "4:\tli\t%1, %3\n\t"                \
 502                         "j\t3b\n\t"                         \
 503                         ".previous\n\t"                     \
 504                         ".section\t__ex_table,\"a\"\n\t"    \
 505                         STR(PTR)"\t1b, 4b\n\t"              \
 506                         STR(PTR)"\t2b, 4b\n\t"              \
 507                         ".previous"                         \
 508                         : "=&r" (value), "=r" (res)         \
 509                         : "r" (addr), "i" (-EFAULT));       \
 510 } while(0)
 511
 512 #ifdef CONFIG_CPU_HAS_LOAD_STORE_LR
 513 #define     _LoadW(addr, value, res, type)   \
 514 do {                                                        \
 515                 __asm__ __volatile__ (                      \
 516                         "1:\t"type##_lwl("%0", "3(%2)")"\n" \
 517                         "2:\t"type##_lwr("%0", "(%2)")"\n\t"\
 518                         "li\t%1, 0\n"                       \
 519                         "3:\n\t"                            \
 520                         ".insn\n\t"                         \
 521                         ".section\t.fixup,\"ax\"\n\t"       \
 522                         "4:\tli\t%1, %3\n\t"                \
 523                         "j\t3b\n\t"                         \
 524                         ".previous\n\t"                     \
 525                         ".section\t__ex_table,\"a\"\n\t"    \
 526                         STR(PTR)"\t1b, 4b\n\t"              \
 527                         STR(PTR)"\t2b, 4b\n\t"              \
 528                         ".previous"                         \
 529                         : "=&r" (value), "=r" (res)         \
 530                         : "r" (addr), "i" (-EFAULT));       \
 531 } while(0)
 532
 533 #else  /* !CONFIG_CPU_HAS_LOAD_STORE_LR */
 534 /* For CPUs without lwl instruction */
 535 #define     _LoadW(addr, value, res, type) \
 536 do {                                                        \
 537                 __asm__ __volatile__ (                      \
 538                         ".set\tpush\n"                      \
 539                         ".set\tnoat\n\t"                    \
 540                         "1:"type##_lb("%0", "3(%2)")"\n\t"  \
 541                         "2:"type##_lbu("$1", "2(%2)")"\n\t" \
 542                         "sll\t%0, 0x8\n\t"                  \
 543                         "or\t%0, $1\n\t"                    \
 544                         "3:"type##_lbu("$1", "1(%2)")"\n\t" \
 545                         "sll\t%0, 0x8\n\t"                  \
 546                         "or\t%0, $1\n\t"                    \
 547                         "4:"type##_lbu("$1", "0(%2)")"\n\t" \
 548                         "sll\t%0, 0x8\n\t"                  \
 549                         "or\t%0, $1\n\t"                    \
 550                         "li\t%1, 0\n"                       \
 551                         ".set\tpop\n"                       \
 552                         "10:\n\t"                           \
 553                         ".insn\n\t"                         \
 554                         ".section\t.fixup,\"ax\"\n\t"       \
 555                         "11:\tli\t%1, %3\n\t"               \
 556                         "j\t10b\n\t"                        \
 557                         ".previous\n\t"                     \
 558                         ".section\t__ex_table,\"a\"\n\t"    \
 559                         STR(PTR)"\t1b, 11b\n\t"             \
 560                         STR(PTR)"\t2b, 11b\n\t"             \
 561                         STR(PTR)"\t3b, 11b\n\t"             \
 562                         STR(PTR)"\t4b, 11b\n\t"             \
 563                         ".previous"                         \
 564                         : "=&r" (value), "=r" (res)         \
 565                         : "r" (addr), "i" (-EFAULT));       \
 566 } while(0)
 567
 568 #endif /* !CONFIG_CPU_HAS_LOAD_STORE_LR */
 569
 570
 571 #define     _LoadHWU(addr, value, res, type) \
 572 do {                                                        \
 573                 __asm__ __volatile__ (                      \
 574                         ".set\tnoat\n"                      \
 575                         "1:\t"type##_lbu("%0", "1(%2)")"\n" \
 576                         "2:\t"type##_lbu("$1", "0(%2)")"\n\t"\
 577                         "sll\t%0, 0x8\n\t"                  \
 578                         "or\t%0, $1\n\t"                    \
 579                         "li\t%1, 0\n"                       \
 580                         "3:\n\t"                            \
 581                         ".insn\n\t"                         \
 582                         ".set\tat\n\t"                      \
 583                         ".section\t.fixup,\"ax\"\n\t"       \
 584                         "4:\tli\t%1, %3\n\t"                \
 585                         "j\t3b\n\t"                         \
 586                         ".previous\n\t"                     \
 587                         ".section\t__ex_table,\"a\"\n\t"    \
 588                         STR(PTR)"\t1b, 4b\n\t"              \
 589                         STR(PTR)"\t2b, 4b\n\t"              \
 590                         ".previous"                         \
 591                         : "=&r" (value), "=r" (res)         \
 592                         : "r" (addr), "i" (-EFAULT));       \
 593 } while(0)
 594
 595 #ifdef CONFIG_CPU_HAS_LOAD_STORE_LR
 596 #define     _LoadWU(addr, value, res, type)  \
 597 do {                                                        \
 598                 __asm__ __volatile__ (                      \
 599                         "1:\t"type##_lwl("%0", "3(%2)")"\n" \
 600                         "2:\t"type##_lwr("%0", "(%2)")"\n\t"\
 601                         "dsll\t%0, %0, 32\n\t"              \
 602                         "dsrl\t%0, %0, 32\n\t"              \
 603                         "li\t%1, 0\n"                       \
 604                         "3:\n\t"                            \
 605                         ".insn\n\t"                         \
 606                         "\t.section\t.fixup,\"ax\"\n\t"     \
 607                         "4:\tli\t%1, %3\n\t"                \
 608                         "j\t3b\n\t"                         \
 609                         ".previous\n\t"                     \
 610                         ".section\t__ex_table,\"a\"\n\t"    \
 611                         STR(PTR)"\t1b, 4b\n\t"              \
 612                         STR(PTR)"\t2b, 4b\n\t"              \
 613                         ".previous"                         \
 614                         : "=&r" (value), "=r" (res)         \
 615                         : "r" (addr), "i" (-EFAULT));       \
 616 } while(0)
 617
 618 #define     _LoadDW(addr, value, res)  \
 619 do {                                                        \
 620                 __asm__ __volatile__ (                      \
 621                         "1:\tldl\t%0, 7(%2)\n"              \
 622                         "2:\tldr\t%0, (%2)\n\t"             \
 623                         "li\t%1, 0\n"                       \
 624                         "3:\n\t"                            \
 625                         ".insn\n\t"                         \
 626                         "\t.section\t.fixup,\"ax\"\n\t"     \
 627                         "4:\tli\t%1, %3\n\t"                \
 628                         "j\t3b\n\t"                         \
 629                         ".previous\n\t"                     \
 630                         ".section\t__ex_table,\"a\"\n\t"    \
 631                         STR(PTR)"\t1b, 4b\n\t"              \
 632                         STR(PTR)"\t2b, 4b\n\t"              \
 633                         ".previous"                         \
 634                         : "=&r" (value), "=r" (res)         \
 635                         : "r" (addr), "i" (-EFAULT));       \
 636 } while(0)
 637
 638 #else /* !CONFIG_CPU_HAS_LOAD_STORE_LR */
 639 /* For CPUs without lwl and ldl instructions */
 640 #define     _LoadWU(addr, value, res, type) \
 641 do {                                                        \
 642                 __asm__ __volatile__ (                      \
 643                         ".set\tpush\n\t"                    \
 644                         ".set\tnoat\n\t"                    \
 645                         "1:"type##_lbu("%0", "3(%2)")"\n\t" \
 646                         "2:"type##_lbu("$1", "2(%2)")"\n\t" \
 647                         "sll\t%0, 0x8\n\t"                  \
 648                         "or\t%0, $1\n\t"                    \
 649                         "3:"type##_lbu("$1", "1(%2)")"\n\t" \
 650                         "sll\t%0, 0x8\n\t"                  \
 651                         "or\t%0, $1\n\t"                    \
 652                         "4:"type##_lbu("$1", "0(%2)")"\n\t" \
 653                         "sll\t%0, 0x8\n\t"                  \
 654                         "or\t%0, $1\n\t"                    \
 655                         "li\t%1, 0\n"                       \
 656                         ".set\tpop\n"                       \
 657                         "10:\n\t"                           \
 658                         ".insn\n\t"                         \
 659                         ".section\t.fixup,\"ax\"\n\t"       \
 660                         "11:\tli\t%1, %3\n\t"               \
 661                         "j\t10b\n\t"                        \
 662                         ".previous\n\t"                     \
 663                         ".section\t__ex_table,\"a\"\n\t"    \
 664                         STR(PTR)"\t1b, 11b\n\t"             \
 665                         STR(PTR)"\t2b, 11b\n\t"             \
 666                         STR(PTR)"\t3b, 11b\n\t"             \
 667                         STR(PTR)"\t4b, 11b\n\t"             \
 668                         ".previous"                         \
 669                         : "=&r" (value), "=r" (res)         \
 670                         : "r" (addr), "i" (-EFAULT));       \
 671 } while(0)
 672
 673 #define     _LoadDW(addr, value, res)  \
 674 do {                                                        \
 675                 __asm__ __volatile__ (                      \
 676                         ".set\tpush\n\t"                    \
 677                         ".set\tnoat\n\t"                    \
 678                         "1:lb\t%0, 7(%2)\n\t"               \
 679                         "2:lbu\t$1, 6(%2)\n\t"              \
 680                         "dsll\t%0, 0x8\n\t"                 \
 681                         "or\t%0, $1\n\t"                    \
 682                         "3:lbu\t$1, 5(%2)\n\t"              \
 683                         "dsll\t%0, 0x8\n\t"                 \
 684                         "or\t%0, $1\n\t"                    \
 685                         "4:lbu\t$1, 4(%2)\n\t"              \
 686                         "dsll\t%0, 0x8\n\t"                 \
 687                         "or\t%0, $1\n\t"                    \
 688                         "5:lbu\t$1, 3(%2)\n\t"              \
 689                         "dsll\t%0, 0x8\n\t"                 \
 690                         "or\t%0, $1\n\t"                    \
 691                         "6:lbu\t$1, 2(%2)\n\t"              \
 692                         "dsll\t%0, 0x8\n\t"                 \
 693                         "or\t%0, $1\n\t"                    \
 694                         "7:lbu\t$1, 1(%2)\n\t"              \
 695                         "dsll\t%0, 0x8\n\t"                 \
 696                         "or\t%0, $1\n\t"                    \
 697                         "8:lbu\t$1, 0(%2)\n\t"              \
 698                         "dsll\t%0, 0x8\n\t"                 \
 699                         "or\t%0, $1\n\t"                    \
 700                         "li\t%1, 0\n"                       \
 701                         ".set\tpop\n\t"                     \
 702                         "10:\n\t"                           \
 703                         ".insn\n\t"                         \
 704                         ".section\t.fixup,\"ax\"\n\t"       \
 705                         "11:\tli\t%1, %3\n\t"               \
 706                         "j\t10b\n\t"                        \
 707                         ".previous\n\t"                     \
 708                         ".section\t__ex_table,\"a\"\n\t"    \
 709                         STR(PTR)"\t1b, 11b\n\t"             \
 710                         STR(PTR)"\t2b, 11b\n\t"             \
 711                         STR(PTR)"\t3b, 11b\n\t"             \
 712                         STR(PTR)"\t4b, 11b\n\t"             \
 713                         STR(PTR)"\t5b, 11b\n\t"             \
 714                         STR(PTR)"\t6b, 11b\n\t"             \
 715                         STR(PTR)"\t7b, 11b\n\t"             \
 716                         STR(PTR)"\t8b, 11b\n\t"             \
 717                         ".previous"                         \
 718                         : "=&r" (value), "=r" (res)         \
 719                         : "r" (addr), "i" (-EFAULT));       \
 720 } while(0)
 721 #endif /* !CONFIG_CPU_HAS_LOAD_STORE_LR */
 722
 723 #define     _StoreHW(addr, value, res, type) \
 724 do {                                                        \
 725                 __asm__ __volatile__ (                      \
 726                         ".set\tnoat\n"                      \
 727                         "1:\t"type##_sb("%1", "0(%2)")"\n"  \
 728                         "srl\t$1,%1, 0x8\n"                 \
 729                         "2:\t"type##_sb("$1", "1(%2)")"\n"  \
 730                         ".set\tat\n\t"                      \
 731                         "li\t%0, 0\n"                       \
 732                         "3:\n\t"                            \
 733                         ".insn\n\t"                         \
 734                         ".section\t.fixup,\"ax\"\n\t"       \
 735                         "4:\tli\t%0, %3\n\t"                \
 736                         "j\t3b\n\t"                         \
 737                         ".previous\n\t"                     \
 738                         ".section\t__ex_table,\"a\"\n\t"    \
 739                         STR(PTR)"\t1b, 4b\n\t"              \
 740                         STR(PTR)"\t2b, 4b\n\t"              \
 741                         ".previous"                         \
 742                         : "=r" (res)                        \
 743                         : "r" (value), "r" (addr), "i" (-EFAULT));\
 744 } while(0)
 745
 746 #ifdef CONFIG_CPU_HAS_LOAD_STORE_LR
 747 #define     _StoreW(addr, value, res, type)  \
 748 do {                                                        \
 749                 __asm__ __volatile__ (                      \
 750                         "1:\t"type##_swl("%1", "3(%2)")"\n" \
 751                         "2:\t"type##_swr("%1", "(%2)")"\n\t"\
 752                         "li\t%0, 0\n"                       \
 753                         "3:\n\t"                            \
 754                         ".insn\n\t"                         \
 755                         ".section\t.fixup,\"ax\"\n\t"       \
 756                         "4:\tli\t%0, %3\n\t"                \
 757                         "j\t3b\n\t"                         \
 758                         ".previous\n\t"                     \
 759                         ".section\t__ex_table,\"a\"\n\t"    \
 760                         STR(PTR)"\t1b, 4b\n\t"              \
 761                         STR(PTR)"\t2b, 4b\n\t"              \
 762                         ".previous"                         \
 763                 : "=r" (res)                                \
 764                 : "r" (value), "r" (addr), "i" (-EFAULT));  \
 765 } while(0)
 766
 767 #define     _StoreDW(addr, value, res) \
 768 do {                                                        \
 769                 __asm__ __volatile__ (                      \
 770                         "1:\tsdl\t%1, 7(%2)\n"              \
 771                         "2:\tsdr\t%1, (%2)\n\t"             \
 772                         "li\t%0, 0\n"                       \
 773                         "3:\n\t"                            \
 774                         ".insn\n\t"                         \
 775                         ".section\t.fixup,\"ax\"\n\t"       \
 776                         "4:\tli\t%0, %3\n\t"                \
 777                         "j\t3b\n\t"                         \
 778                         ".previous\n\t"                     \
 779                         ".section\t__ex_table,\"a\"\n\t"    \
 780                         STR(PTR)"\t1b, 4b\n\t"              \
 781                         STR(PTR)"\t2b, 4b\n\t"              \
 782                         ".previous"                         \
 783                 : "=r" (res)                                \
 784                 : "r" (value), "r" (addr), "i" (-EFAULT));  \
 785 } while(0)
 786
 787 #else /* !CONFIG_CPU_HAS_LOAD_STORE_LR */
 788 /* For CPUs without swl and sdl instructions */
 789 #define     _StoreW(addr, value, res, type)  \
 790 do {                                                        \
 791                 __asm__ __volatile__ (                      \
 792                         ".set\tpush\n\t"                    \
 793                         ".set\tnoat\n\t"                    \
 794                         "1:"type##_sb("%1", "0(%2)")"\n\t"  \
 795                         "srl\t$1, %1, 0x8\n\t"              \
 796                         "2:"type##_sb("$1", "1(%2)")"\n\t"  \
 797                         "srl\t$1, $1,  0x8\n\t"             \
 798                         "3:"type##_sb("$1", "2(%2)")"\n\t"  \
 799                         "srl\t$1, $1, 0x8\n\t"              \
 800                         "4:"type##_sb("$1", "3(%2)")"\n\t"  \
 801                         ".set\tpop\n\t"                     \
 802                         "li\t%0, 0\n"                       \
 803                         "10:\n\t"                           \
 804                         ".insn\n\t"                         \
 805                         ".section\t.fixup,\"ax\"\n\t"       \
 806                         "11:\tli\t%0, %3\n\t"               \
 807                         "j\t10b\n\t"                        \
 808                         ".previous\n\t"                     \
 809                         ".section\t__ex_table,\"a\"\n\t"    \
 810                         STR(PTR)"\t1b, 11b\n\t"             \
 811                         STR(PTR)"\t2b, 11b\n\t"             \
 812                         STR(PTR)"\t3b, 11b\n\t"             \
 813                         STR(PTR)"\t4b, 11b\n\t"             \
 814                         ".previous"                         \
 815                 : "=&r" (res)                               \
 816                 : "r" (value), "r" (addr), "i" (-EFAULT)    \
 817                 : "memory");                                \
 818 } while(0)
 819
 820 #define     _StoreDW(addr, value, res) \
 821 do {                                                        \
 822                 __asm__ __volatile__ (                      \
 823                         ".set\tpush\n\t"                    \
 824                         ".set\tnoat\n\t"                    \
 825                         "1:sb\t%1, 0(%2)\n\t"               \
 826                         "dsrl\t$1, %1, 0x8\n\t"             \
 827                         "2:sb\t$1, 1(%2)\n\t"               \
 828                         "dsrl\t$1, $1, 0x8\n\t"             \
 829                         "3:sb\t$1, 2(%2)\n\t"               \
 830                         "dsrl\t$1, $1, 0x8\n\t"             \
 831                         "4:sb\t$1, 3(%2)\n\t"               \
 832                         "dsrl\t$1, $1, 0x8\n\t"             \
 833                         "5:sb\t$1, 4(%2)\n\t"               \
 834                         "dsrl\t$1, $1, 0x8\n\t"             \
 835                         "6:sb\t$1, 5(%2)\n\t"               \
 836                         "dsrl\t$1, $1, 0x8\n\t"             \
 837                         "7:sb\t$1, 6(%2)\n\t"               \
 838                         "dsrl\t$1, $1, 0x8\n\t"             \
 839                         "8:sb\t$1, 7(%2)\n\t"               \
 840                         "dsrl\t$1, $1, 0x8\n\t"             \
 841                         ".set\tpop\n\t"                     \
 842                         "li\t%0, 0\n"                       \
 843                         "10:\n\t"                           \
 844                         ".insn\n\t"                         \
 845                         ".section\t.fixup,\"ax\"\n\t"       \
 846                         "11:\tli\t%0, %3\n\t"               \
 847                         "j\t10b\n\t"                        \
 848                         ".previous\n\t"                     \
 849                         ".section\t__ex_table,\"a\"\n\t"    \
 850                         STR(PTR)"\t1b, 11b\n\t"             \
 851                         STR(PTR)"\t2b, 11b\n\t"             \
 852                         STR(PTR)"\t3b, 11b\n\t"             \
 853                         STR(PTR)"\t4b, 11b\n\t"             \
 854                         STR(PTR)"\t5b, 11b\n\t"             \
 855                         STR(PTR)"\t6b, 11b\n\t"             \
 856                         STR(PTR)"\t7b, 11b\n\t"             \
 857                         STR(PTR)"\t8b, 11b\n\t"             \
 858                         ".previous"                         \
 859                 : "=&r" (res)                               \
 860                 : "r" (value), "r" (addr), "i" (-EFAULT)    \
 861                 : "memory");                                \
 862 } while(0)
 863
 864 #endif /* !CONFIG_CPU_HAS_LOAD_STORE_LR */
 865 #endif
 866
 867 #define LoadHWU(addr, value, res)       _LoadHWU(addr, value, res, kernel)
 868 #define LoadHWUE(addr, value, res)      _LoadHWU(addr, value, res, user)
 869 #define LoadWU(addr, value, res)        _LoadWU(addr, value, res, kernel)
 870 #define LoadWUE(addr, value, res)       _LoadWU(addr, value, res, user)
 871 #define LoadHW(addr, value, res)        _LoadHW(addr, value, res, kernel)
 872 #define LoadHWE(addr, value, res)       _LoadHW(addr, value, res, user)
 873 #define LoadW(addr, value, res)         _LoadW(addr, value, res, kernel)
 874 #define LoadWE(addr, value, res)        _LoadW(addr, value, res, user)
 875 #define LoadDW(addr, value, res)        _LoadDW(addr, value, res)
 876
 877 #define StoreHW(addr, value, res)       _StoreHW(addr, value, res, kernel)
 878 #define StoreHWE(addr, value, res)      _StoreHW(addr, value, res, user)
 879 #define StoreW(addr, value, res)        _StoreW(addr, value, res, kernel)
 880 #define StoreWE(addr, value, res)       _StoreW(addr, value, res, user)
 881 #define StoreDW(addr, value, res)       _StoreDW(addr, value, res)
 882
 883 static void emulate_load_store_insn(struct pt_regs *regs,
 884         void __user *addr, unsigned int __user *pc)
 885 {
 886         unsigned long origpc, orig31, value;
 887         union mips_instruction insn;
 888         unsigned int res;
 889 #ifdef  CONFIG_EVA
 890         mm_segment_t seg;
 891 #endif
 892         origpc = (unsigned long)pc;
 893         orig31 = regs->regs[31];
 894
 895         perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, 0);
 896
 897         /*
 898          * This load never faults.
 899          */
 900         __get_user(insn.word, pc);
 901
 902         switch (insn.i_format.opcode) {
 903                 /*
 904                  * These are instructions that a compiler doesn't generate.  We
 905                  * can assume therefore that the code is MIPS-aware and
 906                  * really buggy.  Emulating these instructions would break the
 907                  * semantics anyway.
 908                  */
 909         case ll_op:
 910         case lld_op:
 911         case sc_op:
 912         case scd_op:
 913
 914                 /*
 915                  * For these instructions the only way to create an address
 916                  * error is an attempted access to kernel/supervisor address
 917                  * space.
 918                  */
 919         case ldl_op:
 920         case ldr_op:
 921         case lwl_op:
 922         case lwr_op:
 923         case sdl_op:
 924         case sdr_op:
 925         case swl_op:
 926         case swr_op:
 927         case lb_op:
 928         case lbu_op:
 929         case sb_op:
 930                 goto sigbus;
 931
 932                 /*
 933                  * The remaining opcodes are the ones that are really of
 934                  * interest.
 935                  */
 936         case spec3_op:
 937                 if (insn.dsp_format.func == lx_op) {
 938                         switch (insn.dsp_format.op) {
 939                         case lwx_op:
 940                                 if (!access_ok(addr, 4))
 941                                         goto sigbus;
 942                                 LoadW(addr, value, res);
 943                                 if (res)
 944                                         goto fault;
 945                                 compute_return_epc(regs);
 946                                 regs->regs[insn.dsp_format.rd] = value;
 947                                 break;
 948                         case lhx_op:
 949                                 if (!access_ok(addr, 2))
 950                                         goto sigbus;
 951                                 LoadHW(addr, value, res);
 952                                 if (res)
 953                                         goto fault;
 954                                 compute_return_epc(regs);
 955                                 regs->regs[insn.dsp_format.rd] = value;
 956                                 break;
 957                         default:
 958                                 goto sigill;
 959                         }
 960                 }
 961 #ifdef CONFIG_EVA
 962                 else {
 963                         /*
 964                          * we can land here only from kernel accessing user
 965                          * memory, so we need to "switch" the address limit to
 966                          * user space, so that address check can work properly.
 967                          */
 968                         seg = get_fs();
 969                         set_fs(USER_DS);
 970                         switch (insn.spec3_format.func) {
 971                         case lhe_op:
 972                                 if (!access_ok(addr, 2)) {
 973                                         set_fs(seg);
 974                                         goto sigbus;
 975                                 }
 976                                 LoadHWE(addr, value, res);
 977                                 if (res) {
 978                                         set_fs(seg);
 979                                         goto fault;
 980                                 }
 981                                 compute_return_epc(regs);
 982                                 regs->regs[insn.spec3_format.rt] = value;
 983                                 break;
 984                         case lwe_op:
 985                                 if (!access_ok(addr, 4)) {
 986                                         set_fs(seg);
 987                                         goto sigbus;
 988                                 }
 989                                 LoadWE(addr, value, res);
 990                                 if (res) {
 991                                         set_fs(seg);
 992                                         goto fault;
 993                                 }
 994                                 compute_return_epc(regs);
 995                                 regs->regs[insn.spec3_format.rt] = value;
 996                                 break;
 997                         case lhue_op:
 998                                 if (!access_ok(addr, 2)) {
 999                                         set_fs(seg);
1000                                         goto sigbus;
1001                                 }
1002                                 LoadHWUE(addr, value, res);
1003                                 if (res) {
1004                                         set_fs(seg);
1005                                         goto fault;
1006                                 }
1007                                 compute_return_epc(regs);
1008                                 regs->regs[insn.spec3_format.rt] = value;
1009                                 break;
1010                         case she_op:
1011                                 if (!access_ok(addr, 2)) {
1012                                         set_fs(seg);
1013                                         goto sigbus;
1014                                 }
1015                                 compute_return_epc(regs);
1016                                 value = regs->regs[insn.spec3_format.rt];
1017                                 StoreHWE(addr, value, res);
1018                                 if (res) {
1019                                         set_fs(seg);
1020                                         goto fault;
1021                                 }
1022                                 break;
1023                         case swe_op:
1024                                 if (!access_ok(addr, 4)) {
1025                                         set_fs(seg);
1026                                         goto sigbus;
1027                                 }
1028                                 compute_return_epc(regs);
1029                                 value = regs->regs[insn.spec3_format.rt];
1030                                 StoreWE(addr, value, res);
1031                                 if (res) {
1032                                         set_fs(seg);
1033                                         goto fault;
1034                                 }
1035                                 break;
1036                         default:
1037                                 set_fs(seg);
1038                                 goto sigill;
1039                         }
1040                         set_fs(seg);
1041                 }
1042 #endif
1043                 break;
1044         case lh_op:
1045                 if (!access_ok(addr, 2))
1046                         goto sigbus;
1047
1048                 if (IS_ENABLED(CONFIG_EVA)) {
1049                         if (uaccess_kernel())
1050                                 LoadHW(addr, value, res);
1051                         else
1052                                 LoadHWE(addr, value, res);
1053                 } else {
1054                         LoadHW(addr, value, res);
1055                 }
1056
1057                 if (res)
1058                         goto fault;
1059                 compute_return_epc(regs);
1060                 regs->regs[insn.i_format.rt] = value;
1061                 break;
1062
1063         case lw_op:
1064                 if (!access_ok(addr, 4))
1065                         goto sigbus;
1066
1067                 if (IS_ENABLED(CONFIG_EVA)) {
1068                         if (uaccess_kernel())
1069                                 LoadW(addr, value, res);
1070                         else
1071                                 LoadWE(addr, value, res);
1072                 } else {
1073                         LoadW(addr, value, res);
1074                 }
1075
1076                 if (res)
1077                         goto fault;
1078                 compute_return_epc(regs);
1079                 regs->regs[insn.i_format.rt] = value;
1080                 break;
1081
1082         case lhu_op:
1083                 if (!access_ok(addr, 2))
1084                         goto sigbus;
1085
1086                 if (IS_ENABLED(CONFIG_EVA)) {
1087                         if (uaccess_kernel())
1088                                 LoadHWU(addr, value, res);
1089                         else
1090                                 LoadHWUE(addr, value, res);
1091                 } else {
1092                         LoadHWU(addr, value, res);
1093                 }
1094
1095                 if (res)
1096                         goto fault;
1097                 compute_return_epc(regs);
1098                 regs->regs[insn.i_format.rt] = value;
1099                 break;
1100
1101         case lwu_op:
1102 #ifdef CONFIG_64BIT
1103                 /*
1104                  * A 32-bit kernel might be running on a 64-bit processor.  But
1105                  * if we're on a 32-bit processor and an i-cache incoherency
1106                  * or race makes us see a 64-bit instruction here the sdl/sdr
1107                  * would blow up, so for now we don't handle unaligned 64-bit
1108                  * instructions on 32-bit kernels.
1109                  */
1110                 if (!access_ok(addr, 4))
1111                         goto sigbus;
1112
1113                 LoadWU(addr, value, res);
1114                 if (res)
1115                         goto fault;
1116                 compute_return_epc(regs);
1117                 regs->regs[insn.i_format.rt] = value;
1118                 break;
1119 #endif /* CONFIG_64BIT */
1120
1121                 /* Cannot handle 64-bit instructions in 32-bit kernel */
1122                 goto sigill;
1123
1124         case ld_op:
1125 #ifdef CONFIG_64BIT
1126                 /*
1127                  * A 32-bit kernel might be running on a 64-bit processor.  But
1128                  * if we're on a 32-bit processor and an i-cache incoherency
1129                  * or race makes us see a 64-bit instruction here the sdl/sdr
1130                  * would blow up, so for now we don't handle unaligned 64-bit
1131                  * instructions on 32-bit kernels.
1132                  */
1133                 if (!access_ok(addr, 8))
1134                         goto sigbus;
1135
1136                 LoadDW(addr, value, res);
1137                 if (res)
1138                         goto fault;
1139                 compute_return_epc(regs);
1140                 regs->regs[insn.i_format.rt] = value;
1141                 break;
1142 #endif /* CONFIG_64BIT */
1143
1144                 /* Cannot handle 64-bit instructions in 32-bit kernel */
1145                 goto sigill;
1146
1147         case sh_op:
1148                 if (!access_ok(addr, 2))
1149                         goto sigbus;
1150
1151                 compute_return_epc(regs);
1152                 value = regs->regs[insn.i_format.rt];
1153
1154                 if (IS_ENABLED(CONFIG_EVA)) {
1155                         if (uaccess_kernel())
1156                                 StoreHW(addr, value, res);
1157                         else
1158                                 StoreHWE(addr, value, res);
1159                 } else {
1160                         StoreHW(addr, value, res);
1161                 }
1162
1163                 if (res)
1164                         goto fault;
1165                 break;
1166
1167         case sw_op:
1168                 if (!access_ok(addr, 4))
1169                         goto sigbus;
1170
1171                 compute_return_epc(regs);
1172                 value = regs->regs[insn.i_format.rt];
1173
1174                 if (IS_ENABLED(CONFIG_EVA)) {
1175                         if (uaccess_kernel())
1176                                 StoreW(addr, value, res);
1177                         else
1178                                 StoreWE(addr, value, res);
1179                 } else {
1180                         StoreW(addr, value, res);
1181                 }
1182
1183                 if (res)
1184                         goto fault;
1185                 break;
1186
1187         case sd_op:
1188 #ifdef CONFIG_64BIT
1189                 /*
1190                  * A 32-bit kernel might be running on a 64-bit processor.  But
1191                  * if we're on a 32-bit processor and an i-cache incoherency
1192                  * or race makes us see a 64-bit instruction here the sdl/sdr
1193                  * would blow up, so for now we don't handle unaligned 64-bit
1194                  * instructions on 32-bit kernels.
1195                  */
1196                 if (!access_ok(addr, 8))
1197                         goto sigbus;
1198
1199                 compute_return_epc(regs);
1200                 value = regs->regs[insn.i_format.rt];
1201                 StoreDW(addr, value, res);
1202                 if (res)
1203                         goto fault;
1204                 break;
1205 #endif /* CONFIG_64BIT */
1206
1207                 /* Cannot handle 64-bit instructions in 32-bit kernel */
1208                 goto sigill;
1209
1210 #ifdef CONFIG_MIPS_FP_SUPPORT
1211
1212         case lwc1_op:
1213         case ldc1_op:
1214         case swc1_op:
1215         case sdc1_op:
1216         case cop1x_op: {
1217                 void __user *fault_addr = NULL;
1218
1219                 die_if_kernel("Unaligned FP access in kernel code", regs);
1220                 BUG_ON(!used_math());
1221
1222                 res = fpu_emulator_cop1Handler(regs, &current->thread.fpu, 1,
1223                                                &fault_addr);
1224                 own_fpu(1);     /* Restore FPU state. */
1225
1226                 /* Signal if something went wrong. */
1227                 process_fpemu_return(res, fault_addr, 0);
1228
1229                 if (res == 0)
1230                         break;
1231                 return;
1232         }
1233 #endif /* CONFIG_MIPS_FP_SUPPORT */
1234
1235 #ifdef CONFIG_CPU_HAS_MSA
1236
1237         case msa_op: {
1238                 unsigned int wd, preempted;
1239                 enum msa_2b_fmt df;
1240                 union fpureg *fpr;
1241
1242                 if (!cpu_has_msa)
1243                         goto sigill;
1244
1245                 /*
1246                  * If we've reached this point then userland should have taken
1247                  * the MSA disabled exception & initialised vector context at
1248                  * some point in the past.
1249                  */
1250                 BUG_ON(!thread_msa_context_live());
1251
1252                 df = insn.msa_mi10_format.df;
1253                 wd = insn.msa_mi10_format.wd;
1254                 fpr = &current->thread.fpu.fpr[wd];
1255
1256                 switch (insn.msa_mi10_format.func) {
1257                 case msa_ld_op:
1258                         if (!access_ok(addr, sizeof(*fpr)))
1259                                 goto sigbus;
1260
1261                         do {
1262                                 /*
1263                                  * If we have live MSA context keep track of
1264                                  * whether we get preempted in order to avoid
1265                                  * the register context we load being clobbered
1266                                  * by the live context as it's saved during
1267                                  * preemption. If we don't have live context
1268                                  * then it can't be saved to clobber the value
1269                                  * we load.
1270                                  */
1271                                 preempted = test_thread_flag(TIF_USEDMSA);
1272
1273                                 res = __copy_from_user_inatomic(fpr, addr,
1274                                                                 sizeof(*fpr));
1275                                 if (res)
1276                                         goto fault;
1277
1278                                 /*
1279                                  * Update the hardware register if it is in use
1280                                  * by the task in this quantum, in order to
1281                                  * avoid having to save & restore the whole
1282                                  * vector context.
1283                                  */
1284                                 preempt_disable();
1285                                 if (test_thread_flag(TIF_USEDMSA)) {
1286                                         write_msa_wr(wd, fpr, df);
1287                                         preempted = 0;
1288                                 }
1289                                 preempt_enable();
1290                         } while (preempted);
1291                         break;
1292
1293                 case msa_st_op:
1294                         if (!access_ok(addr, sizeof(*fpr)))
1295                                 goto sigbus;
1296
1297                         /*
1298                          * Update from the hardware register if it is in use by
1299                          * the task in this quantum, in order to avoid having to
1300                          * save & restore the whole vector context.
1301                          */
1302                         preempt_disable();
1303                         if (test_thread_flag(TIF_USEDMSA))
1304                                 read_msa_wr(wd, fpr, df);
1305                         preempt_enable();
1306
1307                         res = __copy_to_user_inatomic(addr, fpr, sizeof(*fpr));
1308                         if (res)
1309                                 goto fault;
1310                         break;
1311
1312                 default:
1313                         goto sigbus;
1314                 }
1315
1316                 compute_return_epc(regs);
1317                 break;
1318         }
1319 #endif /* CONFIG_CPU_HAS_MSA */
1320
1321 #ifndef CONFIG_CPU_MIPSR6
1322         /*
1323          * COP2 is available to implementor for application specific use.
1324          * It's up to applications to register a notifier chain and do
1325          * whatever they have to do, including possible sending of signals.
1326          *
1327          * This instruction has been reallocated in Release 6
1328          */
1329         case lwc2_op:
1330                 cu2_notifier_call_chain(CU2_LWC2_OP, regs);
1331                 break;
1332
1333         case ldc2_op:
1334                 cu2_notifier_call_chain(CU2_LDC2_OP, regs);
1335                 break;
1336
1337         case swc2_op:
1338                 cu2_notifier_call_chain(CU2_SWC2_OP, regs);
1339                 break;
1340
1341         case sdc2_op:
1342                 cu2_notifier_call_chain(CU2_SDC2_OP, regs);
1343                 break;
1344 #endif
1345         default:
1346                 /*
1347                  * Pheeee...  We encountered an yet unknown instruction or
1348                  * cache coherence problem.  Die sucker, die ...
1349                  */
1350                 goto sigill;
1351         }
1352
1353 #ifdef CONFIG_DEBUG_FS
1354         unaligned_instructions++;
1355 #endif
1356
1357         return;
1358
1359 fault:
1360         /* roll back jump/branch */
1361         regs->cp0_epc = origpc;
1362         regs->regs[31] = orig31;
1363         /* Did we have an exception handler installed? */
1364         if (fixup_exception(regs))
1365                 return;
1366
1367         die_if_kernel("Unhandled kernel unaligned access", regs);
1368         force_sig(SIGSEGV);
1369
1370         return;
1371
1372 sigbus:
1373         die_if_kernel("Unhandled kernel unaligned access", regs);
1374         force_sig(SIGBUS);
1375
1376         return;
1377
1378 sigill:
1379         die_if_kernel
1380             ("Unhandled kernel unaligned access or invalid instruction", regs);
1381         force_sig(SIGILL);
1382 }
1383
1384 /* Recode table from 16-bit register notation to 32-bit GPR. */
1385 const int reg16to32[] = { 16, 17, 2, 3, 4, 5, 6, 7 };
1386
1387 /* Recode table from 16-bit STORE register notation to 32-bit GPR. */
1388 static const int reg16to32st[] = { 0, 17, 2, 3, 4, 5, 6, 7 };
1389
1390 static void emulate_load_store_microMIPS(struct pt_regs *regs,
1391                                          void __user *addr)
1392 {
1393         unsigned long value;
1394         unsigned int res;
1395         int i;
1396         unsigned int reg = 0, rvar;
1397         unsigned long orig31;
1398         u16 __user *pc16;
1399         u16 halfword;
1400         unsigned int word;
1401         unsigned long origpc, contpc;
1402         union mips_instruction insn;
1403         struct mm_decoded_insn mminsn;
1404
1405         origpc = regs->cp0_epc;
1406         orig31 = regs->regs[31];
1407
1408         mminsn.micro_mips_mode = 1;
1409
1410         /*
1411          * This load never faults.
1412          */
1413         pc16 = (unsigned short __user *)msk_isa16_mode(regs->cp0_epc);
1414         __get_user(halfword, pc16);
1415         pc16++;
1416         contpc = regs->cp0_epc + 2;
1417         word = ((unsigned int)halfword << 16);
1418         mminsn.pc_inc = 2;
1419
1420         if (!mm_insn_16bit(halfword)) {
1421                 __get_user(halfword, pc16);
1422                 pc16++;
1423                 contpc = regs->cp0_epc + 4;
1424                 mminsn.pc_inc = 4;
1425                 word |= halfword;
1426         }
1427         mminsn.insn = word;
1428
1429         if (get_user(halfword, pc16))
1430                 goto fault;
1431         mminsn.next_pc_inc = 2;
1432         word = ((unsigned int)halfword << 16);
1433
1434         if (!mm_insn_16bit(halfword)) {
1435                 pc16++;
1436                 if (get_user(halfword, pc16))
1437                         goto fault;
1438                 mminsn.next_pc_inc = 4;
1439                 word |= halfword;
1440         }
1441         mminsn.next_insn = word;
1442
1443         insn = (union mips_instruction)(mminsn.insn);
1444         if (mm_isBranchInstr(regs, mminsn, &contpc))
1445                 insn = (union mips_instruction)(mminsn.next_insn);
1446
1447         /*  Parse instruction to find what to do */
1448
1449         switch (insn.mm_i_format.opcode) {
1450
1451         case mm_pool32a_op:
1452                 switch (insn.mm_x_format.func) {
1453                 case mm_lwxs_op:
1454                         reg = insn.mm_x_format.rd;
1455                         goto loadW;
1456                 }
1457
1458                 goto sigbus;
1459
1460         case mm_pool32b_op:
1461                 switch (insn.mm_m_format.func) {
1462                 case mm_lwp_func:
1463                         reg = insn.mm_m_format.rd;
1464                         if (reg == 31)
1465                                 goto sigbus;
1466
1467                         if (!access_ok(addr, 8))
1468                                 goto sigbus;
1469
1470                         LoadW(addr, value, res);
1471                         if (res)
1472                                 goto fault;
1473                         regs->regs[reg] = value;
1474                         addr += 4;
1475                         LoadW(addr, value, res);
1476                         if (res)
1477                                 goto fault;
1478                         regs->regs[reg + 1] = value;
1479                         goto success;
1480
1481                 case mm_swp_func:
1482                         reg = insn.mm_m_format.rd;
1483                         if (reg == 31)
1484                                 goto sigbus;
1485
1486                         if (!access_ok(addr, 8))
1487                                 goto sigbus;
1488
1489                         value = regs->regs[reg];
1490                         StoreW(addr, value, res);
1491                         if (res)
1492                                 goto fault;
1493                         addr += 4;
1494                         value = regs->regs[reg + 1];
1495                         StoreW(addr, value, res);
1496                         if (res)
1497                                 goto fault;
1498                         goto success;
1499
1500                 case mm_ldp_func:
1501 #ifdef CONFIG_64BIT
1502                         reg = insn.mm_m_format.rd;
1503                         if (reg == 31)
1504                                 goto sigbus;
1505
1506                         if (!access_ok(addr, 16))
1507                                 goto sigbus;
1508
1509                         LoadDW(addr, value, res);
1510                         if (res)
1511                                 goto fault;
1512                         regs->regs[reg] = value;
1513                         addr += 8;
1514                         LoadDW(addr, value, res);
1515                         if (res)
1516                                 goto fault;
1517                         regs->regs[reg + 1] = value;
1518                         goto success;
1519 #endif /* CONFIG_64BIT */
1520
1521                         goto sigill;
1522
1523                 case mm_sdp_func:
1524 #ifdef CONFIG_64BIT
1525                         reg = insn.mm_m_format.rd;
1526                         if (reg == 31)
1527                                 goto sigbus;
1528
1529                         if (!access_ok(addr, 16))
1530                                 goto sigbus;
1531
1532                         value = regs->regs[reg];
1533                         StoreDW(addr, value, res);
1534                         if (res)
1535                                 goto fault;
1536                         addr += 8;
1537                         value = regs->regs[reg + 1];
1538                         StoreDW(addr, value, res);
1539                         if (res)
1540                                 goto fault;
1541                         goto success;
1542 #endif /* CONFIG_64BIT */
1543
1544                         goto sigill;
1545
1546                 case mm_lwm32_func:
1547                         reg = insn.mm_m_format.rd;
1548                         rvar = reg & 0xf;
1549                         if ((rvar > 9) || !reg)
1550                                 goto sigill;
1551                         if (reg & 0x10) {
1552                                 if (!access_ok(addr, 4 * (rvar + 1)))
1553                                         goto sigbus;
1554                         } else {
1555                                 if (!access_ok(addr, 4 * rvar))
1556                                         goto sigbus;
1557                         }
1558                         if (rvar == 9)
1559                                 rvar = 8;
1560                         for (i = 16; rvar; rvar--, i++) {
1561                                 LoadW(addr, value, res);
1562                                 if (res)
1563                                         goto fault;
1564                                 addr += 4;
1565                                 regs->regs[i] = value;
1566                         }
1567                         if ((reg & 0xf) == 9) {
1568                                 LoadW(addr, value, res);
1569                                 if (res)
1570                                         goto fault;
1571                                 addr += 4;
1572                                 regs->regs[30] = value;
1573                         }
1574                         if (reg & 0x10) {
1575                                 LoadW(addr, value, res);
1576                                 if (res)
1577                                         goto fault;
1578                                 regs->regs[31] = value;
1579                         }
1580                         goto success;
1581
1582                 case mm_swm32_func:
1583                         reg = insn.mm_m_format.rd;
1584                         rvar = reg & 0xf;
1585                         if ((rvar > 9) || !reg)
1586                                 goto sigill;
1587                         if (reg & 0x10) {
1588                                 if (!access_ok(addr, 4 * (rvar + 1)))
1589                                         goto sigbus;
1590                         } else {
1591                                 if (!access_ok(addr, 4 * rvar))
1592                                         goto sigbus;
1593                         }
1594                         if (rvar == 9)
1595                                 rvar = 8;
1596                         for (i = 16; rvar; rvar--, i++) {
1597                                 value = regs->regs[i];
1598                                 StoreW(addr, value, res);
1599                                 if (res)
1600                                         goto fault;
1601                                 addr += 4;
1602                         }
1603                         if ((reg & 0xf) == 9) {
1604                                 value = regs->regs[30];
1605                                 StoreW(addr, value, res);
1606                                 if (res)
1607                                         goto fault;
1608                                 addr += 4;
1609                         }
1610                         if (reg & 0x10) {
1611                                 value = regs->regs[31];
1612                                 StoreW(addr, value, res);
1613                                 if (res)
1614                                         goto fault;
1615                         }
1616                         goto success;
1617
1618                 case mm_ldm_func:
1619 #ifdef CONFIG_64BIT
1620                         reg = insn.mm_m_format.rd;
1621                         rvar = reg & 0xf;
1622                         if ((rvar > 9) || !reg)
1623                                 goto sigill;
1624                         if (reg & 0x10) {
1625                                 if (!access_ok(addr, 8 * (rvar + 1)))
1626                                         goto sigbus;
1627                         } else {
1628                                 if (!access_ok(addr, 8 * rvar))
1629                                         goto sigbus;
1630                         }
1631                         if (rvar == 9)
1632                                 rvar = 8;
1633
1634                         for (i = 16; rvar; rvar--, i++) {
1635                                 LoadDW(addr, value, res);
1636                                 if (res)
1637                                         goto fault;
1638                                 addr += 4;
1639                                 regs->regs[i] = value;
1640                         }
1641                         if ((reg & 0xf) == 9) {
1642                                 LoadDW(addr, value, res);
1643                                 if (res)
1644                                         goto fault;
1645                                 addr += 8;
1646                                 regs->regs[30] = value;
1647                         }
1648                         if (reg & 0x10) {
1649                                 LoadDW(addr, value, res);
1650                                 if (res)
1651                                         goto fault;
1652                                 regs->regs[31] = value;
1653                         }
1654                         goto success;
1655 #endif /* CONFIG_64BIT */
1656
1657                         goto sigill;
1658
1659                 case mm_sdm_func:
1660 #ifdef CONFIG_64BIT
1661                         reg = insn.mm_m_format.rd;
1662                         rvar = reg & 0xf;
1663                         if ((rvar > 9) || !reg)
1664                                 goto sigill;
1665                         if (reg & 0x10) {
1666                                 if (!access_ok(addr, 8 * (rvar + 1)))
1667                                         goto sigbus;
1668                         } else {
1669                                 if (!access_ok(addr, 8 * rvar))
1670                                         goto sigbus;
1671                         }
1672                         if (rvar == 9)
1673                                 rvar = 8;
1674
1675                         for (i = 16; rvar; rvar--, i++) {
1676                                 value = regs->regs[i];
1677                                 StoreDW(addr, value, res);
1678                                 if (res)
1679                                         goto fault;
1680                                 addr += 8;
1681                         }
1682                         if ((reg & 0xf) == 9) {
1683                                 value = regs->regs[30];
1684                                 StoreDW(addr, value, res);
1685                                 if (res)
1686                                         goto fault;
1687                                 addr += 8;
1688                         }
1689                         if (reg & 0x10) {
1690                                 value = regs->regs[31];
1691                                 StoreDW(addr, value, res);
1692                                 if (res)
1693                                         goto fault;
1694                         }
1695                         goto success;
1696 #endif /* CONFIG_64BIT */
1697
1698                         goto sigill;
1699
1700                         /*  LWC2, SWC2, LDC2, SDC2 are not serviced */
1701                 }
1702
1703                 goto sigbus;
1704
1705         case mm_pool32c_op:
1706                 switch (insn.mm_m_format.func) {
1707                 case mm_lwu_func:
1708                         reg = insn.mm_m_format.rd;
1709                         goto loadWU;
1710                 }
1711
1712                 /*  LL,SC,LLD,SCD are not serviced */
1713                 goto sigbus;
1714
1715 #ifdef CONFIG_MIPS_FP_SUPPORT
1716         case mm_pool32f_op:
1717                 switch (insn.mm_x_format.func) {
1718                 case mm_lwxc1_func:
1719                 case mm_swxc1_func:
1720                 case mm_ldxc1_func:
1721                 case mm_sdxc1_func:
1722                         goto fpu_emul;
1723                 }
1724
1725                 goto sigbus;
1726
1727         case mm_ldc132_op:
1728         case mm_sdc132_op:
1729         case mm_lwc132_op:
1730         case mm_swc132_op: {
1731                 void __user *fault_addr = NULL;
1732
1733 fpu_emul:
1734                 /* roll back jump/branch */
1735                 regs->cp0_epc = origpc;
1736                 regs->regs[31] = orig31;
1737
1738                 die_if_kernel("Unaligned FP access in kernel code", regs);
1739                 BUG_ON(!used_math());
1740                 BUG_ON(!is_fpu_owner());
1741
1742                 res = fpu_emulator_cop1Handler(regs, &current->thread.fpu, 1,
1743                                                &fault_addr);
1744                 own_fpu(1);     /* restore FPU state */
1745
1746                 /* If something went wrong, signal */
1747                 process_fpemu_return(res, fault_addr, 0);
1748
1749                 if (res == 0)
1750                         goto success;
1751                 return;
1752         }
1753 #endif /* CONFIG_MIPS_FP_SUPPORT */
1754
1755         case mm_lh32_op:
1756                 reg = insn.mm_i_format.rt;
1757                 goto loadHW;
1758
1759         case mm_lhu32_op:
1760                 reg = insn.mm_i_format.rt;
1761                 goto loadHWU;
1762
1763         case mm_lw32_op:
1764                 reg = insn.mm_i_format.rt;
1765                 goto loadW;
1766
1767         case mm_sh32_op:
1768                 reg = insn.mm_i_format.rt;
1769                 goto storeHW;
1770
1771         case mm_sw32_op:
1772                 reg = insn.mm_i_format.rt;
1773                 goto storeW;
1774
1775         case mm_ld32_op:
1776                 reg = insn.mm_i_format.rt;
1777                 goto loadDW;
1778
1779         case mm_sd32_op:
1780                 reg = insn.mm_i_format.rt;
1781                 goto storeDW;
1782
1783         case mm_pool16c_op:
1784                 switch (insn.mm16_m_format.func) {
1785                 case mm_lwm16_op:
1786                         reg = insn.mm16_m_format.rlist;
1787                         rvar = reg + 1;
1788                         if (!access_ok(addr, 4 * rvar))
1789                                 goto sigbus;
1790
1791                         for (i = 16; rvar; rvar--, i++) {
1792                                 LoadW(addr, value, res);
1793                                 if (res)
1794                                         goto fault;
1795                                 addr += 4;
1796                                 regs->regs[i] = value;
1797                         }
1798                         LoadW(addr, value, res);
1799                         if (res)
1800                                 goto fault;
1801                         regs->regs[31] = value;
1802
1803                         goto success;
1804
1805                 case mm_swm16_op:
1806                         reg = insn.mm16_m_format.rlist;
1807                         rvar = reg + 1;
1808                         if (!access_ok(addr, 4 * rvar))
1809                                 goto sigbus;
1810
1811                         for (i = 16; rvar; rvar--, i++) {
1812                                 value = regs->regs[i];
1813                                 StoreW(addr, value, res);
1814                                 if (res)
1815                                         goto fault;
1816                                 addr += 4;
1817                         }
1818                         value = regs->regs[31];
1819                         StoreW(addr, value, res);
1820                         if (res)
1821                                 goto fault;
1822
1823                         goto success;
1824
1825                 }
1826
1827                 goto sigbus;
1828
1829         case mm_lhu16_op:
1830                 reg = reg16to32[insn.mm16_rb_format.rt];
1831                 goto loadHWU;
1832
1833         case mm_lw16_op:
1834                 reg = reg16to32[insn.mm16_rb_format.rt];
1835                 goto loadW;
1836
1837         case mm_sh16_op:
1838                 reg = reg16to32st[insn.mm16_rb_format.rt];
1839                 goto storeHW;
1840
1841         case mm_sw16_op:
1842                 reg = reg16to32st[insn.mm16_rb_format.rt];
1843                 goto storeW;
1844
1845         case mm_lwsp16_op:
1846                 reg = insn.mm16_r5_format.rt;
1847                 goto loadW;
1848
1849         case mm_swsp16_op:
1850                 reg = insn.mm16_r5_format.rt;
1851                 goto storeW;
1852
1853         case mm_lwgp16_op:
1854                 reg = reg16to32[insn.mm16_r3_format.rt];
1855                 goto loadW;
1856
1857         default:
1858                 goto sigill;
1859         }
1860
1861 loadHW:
1862         if (!access_ok(addr, 2))
1863                 goto sigbus;
1864
1865         LoadHW(addr, value, res);
1866         if (res)
1867                 goto fault;
1868         regs->regs[reg] = value;
1869         goto success;
1870
1871 loadHWU:
1872         if (!access_ok(addr, 2))
1873                 goto sigbus;
1874
1875         LoadHWU(addr, value, res);
1876         if (res)
1877                 goto fault;
1878         regs->regs[reg] = value;
1879         goto success;
1880
1881 loadW:
1882         if (!access_ok(addr, 4))
1883                 goto sigbus;
1884
1885         LoadW(addr, value, res);
1886         if (res)
1887                 goto fault;
1888         regs->regs[reg] = value;
1889         goto success;
1890
1891 loadWU:
1892 #ifdef CONFIG_64BIT
1893         /*
1894          * A 32-bit kernel might be running on a 64-bit processor.  But
1895          * if we're on a 32-bit processor and an i-cache incoherency
1896          * or race makes us see a 64-bit instruction here the sdl/sdr
1897          * would blow up, so for now we don't handle unaligned 64-bit
1898          * instructions on 32-bit kernels.
1899          */
1900         if (!access_ok(addr, 4))
1901                 goto sigbus;
1902
1903         LoadWU(addr, value, res);
1904         if (res)
1905                 goto fault;
1906         regs->regs[reg] = value;
1907         goto success;
1908 #endif /* CONFIG_64BIT */
1909
1910         /* Cannot handle 64-bit instructions in 32-bit kernel */
1911         goto sigill;
1912
1913 loadDW:
1914 #ifdef CONFIG_64BIT
1915         /*
1916          * A 32-bit kernel might be running on a 64-bit processor.  But
1917          * if we're on a 32-bit processor and an i-cache incoherency
1918          * or race makes us see a 64-bit instruction here the sdl/sdr
1919          * would blow up, so for now we don't handle unaligned 64-bit
1920          * instructions on 32-bit kernels.
1921          */
1922         if (!access_ok(addr, 8))
1923                 goto sigbus;
1924
1925         LoadDW(addr, value, res);
1926         if (res)
1927                 goto fault;
1928         regs->regs[reg] = value;
1929         goto success;
1930 #endif /* CONFIG_64BIT */
1931
1932         /* Cannot handle 64-bit instructions in 32-bit kernel */
1933         goto sigill;
1934
1935 storeHW:
1936         if (!access_ok(addr, 2))
1937                 goto sigbus;
1938
1939         value = regs->regs[reg];
1940         StoreHW(addr, value, res);
1941         if (res)
1942                 goto fault;
1943         goto success;
1944
1945 storeW:
1946         if (!access_ok(addr, 4))
1947                 goto sigbus;
1948
1949         value = regs->regs[reg];
1950         StoreW(addr, value, res);
1951         if (res)
1952                 goto fault;
1953         goto success;
1954
1955 storeDW:
1956 #ifdef CONFIG_64BIT
1957         /*
1958          * A 32-bit kernel might be running on a 64-bit processor.  But
1959          * if we're on a 32-bit processor and an i-cache incoherency
1960          * or race makes us see a 64-bit instruction here the sdl/sdr
1961          * would blow up, so for now we don't handle unaligned 64-bit
1962          * instructions on 32-bit kernels.
1963          */
1964         if (!access_ok(addr, 8))
1965                 goto sigbus;
1966
1967         value = regs->regs[reg];
1968         StoreDW(addr, value, res);
1969         if (res)
1970                 goto fault;
1971         goto success;
1972 #endif /* CONFIG_64BIT */
1973
1974         /* Cannot handle 64-bit instructions in 32-bit kernel */
1975         goto sigill;
1976
1977 success:
1978         regs->cp0_epc = contpc; /* advance or branch */
1979
1980 #ifdef CONFIG_DEBUG_FS
1981         unaligned_instructions++;
1982 #endif
1983         return;
1984
1985 fault:
1986         /* roll back jump/branch */
1987         regs->cp0_epc = origpc;
1988         regs->regs[31] = orig31;
1989         /* Did we have an exception handler installed? */
1990         if (fixup_exception(regs))
1991                 return;
1992
1993         die_if_kernel("Unhandled kernel unaligned access", regs);
1994         force_sig(SIGSEGV);
1995
1996         return;
1997
1998 sigbus:
1999         die_if_kernel("Unhandled kernel unaligned access", regs);
2000         force_sig(SIGBUS);
2001
2002         return;
2003
2004 sigill:
2005         die_if_kernel
2006             ("Unhandled kernel unaligned access or invalid instruction", regs);
2007         force_sig(SIGILL);
2008 }
2009
2010 static void emulate_load_store_MIPS16e(struct pt_regs *regs, void __user * addr)
2011 {
2012         unsigned long value;
2013         unsigned int res;
2014         int reg;
2015         unsigned long orig31;
2016         u16 __user *pc16;
2017         unsigned long origpc;
2018         union mips16e_instruction mips16inst, oldinst;
2019         unsigned int opcode;
2020         int extended = 0;
2021
2022         origpc = regs->cp0_epc;
2023         orig31 = regs->regs[31];
2024         pc16 = (unsigned short __user *)msk_isa16_mode(origpc);
2025         /*
2026          * This load never faults.
2027          */
2028         __get_user(mips16inst.full, pc16);
2029         oldinst = mips16inst;
2030
2031         /* skip EXTEND instruction */
2032         if (mips16inst.ri.opcode == MIPS16e_extend_op) {
2033                 extended = 1;
2034                 pc16++;
2035                 __get_user(mips16inst.full, pc16);
2036         } else if (delay_slot(regs)) {
2037                 /*  skip jump instructions */
2038                 /*  JAL/JALX are 32 bits but have OPCODE in first short int */
2039                 if (mips16inst.ri.opcode == MIPS16e_jal_op)
2040                         pc16++;
2041                 pc16++;
2042                 if (get_user(mips16inst.full, pc16))
2043                         goto sigbus;
2044         }
2045
2046         opcode = mips16inst.ri.opcode;
2047         switch (opcode) {
2048         case MIPS16e_i64_op:    /* I64 or RI64 instruction */
2049                 switch (mips16inst.i64.func) {  /* I64/RI64 func field check */
2050                 case MIPS16e_ldpc_func:
2051                 case MIPS16e_ldsp_func:
2052                         reg = reg16to32[mips16inst.ri64.ry];
2053                         goto loadDW;
2054
2055                 case MIPS16e_sdsp_func:
2056                         reg = reg16to32[mips16inst.ri64.ry];
2057                         goto writeDW;
2058
2059                 case MIPS16e_sdrasp_func:
2060                         reg = 29;       /* GPRSP */
2061                         goto writeDW;
2062                 }
2063
2064                 goto sigbus;
2065
2066         case MIPS16e_swsp_op:
2067                 reg = reg16to32[mips16inst.ri.rx];
2068                 if (extended && cpu_has_mips16e2)
2069                         switch (mips16inst.ri.imm >> 5) {
2070                         case 0:         /* SWSP */
2071                         case 1:         /* SWGP */
2072                                 break;
2073                         case 2:         /* SHGP */
2074                                 opcode = MIPS16e_sh_op;
2075                                 break;
2076                         default:
2077                                 goto sigbus;
2078                         }
2079                 break;
2080
2081         case MIPS16e_lwpc_op:
2082                 reg = reg16to32[mips16inst.ri.rx];
2083                 break;
2084
2085         case MIPS16e_lwsp_op:
2086                 reg = reg16to32[mips16inst.ri.rx];
2087                 if (extended && cpu_has_mips16e2)
2088                         switch (mips16inst.ri.imm >> 5) {
2089                         case 0:         /* LWSP */
2090                         case 1:         /* LWGP */
2091                                 break;
2092                         case 2:         /* LHGP */
2093                                 opcode = MIPS16e_lh_op;
2094                                 break;
2095                         case 4:         /* LHUGP */
2096                                 opcode = MIPS16e_lhu_op;
2097                                 break;
2098                         default:
2099                                 goto sigbus;
2100                         }
2101                 break;
2102
2103         case MIPS16e_i8_op:
2104                 if (mips16inst.i8.func != MIPS16e_swrasp_func)
2105                         goto sigbus;
2106                 reg = 29;       /* GPRSP */
2107                 break;
2108
2109         default:
2110                 reg = reg16to32[mips16inst.rri.ry];
2111                 break;
2112         }
2113
2114         switch (opcode) {
2115
2116         case MIPS16e_lb_op:
2117         case MIPS16e_lbu_op:
2118         case MIPS16e_sb_op:
2119                 goto sigbus;
2120
2121         case MIPS16e_lh_op:
2122                 if (!access_ok(addr, 2))
2123                         goto sigbus;
2124
2125                 LoadHW(addr, value, res);
2126                 if (res)
2127                         goto fault;
2128                 MIPS16e_compute_return_epc(regs, &oldinst);
2129                 regs->regs[reg] = value;
2130                 break;
2131
2132         case MIPS16e_lhu_op:
2133                 if (!access_ok(addr, 2))
2134                         goto sigbus;
2135
2136                 LoadHWU(addr, value, res);
2137                 if (res)
2138                         goto fault;
2139                 MIPS16e_compute_return_epc(regs, &oldinst);
2140                 regs->regs[reg] = value;
2141                 break;
2142
2143         case MIPS16e_lw_op:
2144         case MIPS16e_lwpc_op:
2145         case MIPS16e_lwsp_op:
2146                 if (!access_ok(addr, 4))
2147                         goto sigbus;
2148
2149                 LoadW(addr, value, res);
2150                 if (res)
2151                         goto fault;
2152                 MIPS16e_compute_return_epc(regs, &oldinst);
2153                 regs->regs[reg] = value;
2154                 break;
2155
2156         case MIPS16e_lwu_op:
2157 #ifdef CONFIG_64BIT
2158                 /*
2159                  * A 32-bit kernel might be running on a 64-bit processor.  But
2160                  * if we're on a 32-bit processor and an i-cache incoherency
2161                  * or race makes us see a 64-bit instruction here the sdl/sdr
2162                  * would blow up, so for now we don't handle unaligned 64-bit
2163                  * instructions on 32-bit kernels.
2164                  */
2165                 if (!access_ok(addr, 4))
2166                         goto sigbus;
2167
2168                 LoadWU(addr, value, res);
2169                 if (res)
2170                         goto fault;
2171                 MIPS16e_compute_return_epc(regs, &oldinst);
2172                 regs->regs[reg] = value;
2173                 break;
2174 #endif /* CONFIG_64BIT */
2175
2176                 /* Cannot handle 64-bit instructions in 32-bit kernel */
2177                 goto sigill;
2178
2179         case MIPS16e_ld_op:
2180 loadDW:
2181 #ifdef CONFIG_64BIT
2182                 /*
2183                  * A 32-bit kernel might be running on a 64-bit processor.  But
2184                  * if we're on a 32-bit processor and an i-cache incoherency
2185                  * or race makes us see a 64-bit instruction here the sdl/sdr
2186                  * would blow up, so for now we don't handle unaligned 64-bit
2187                  * instructions on 32-bit kernels.
2188                  */
2189                 if (!access_ok(addr, 8))
2190                         goto sigbus;
2191
2192                 LoadDW(addr, value, res);
2193                 if (res)
2194                         goto fault;
2195                 MIPS16e_compute_return_epc(regs, &oldinst);
2196                 regs->regs[reg] = value;
2197                 break;
2198 #endif /* CONFIG_64BIT */
2199
2200                 /* Cannot handle 64-bit instructions in 32-bit kernel */
2201                 goto sigill;
2202
2203         case MIPS16e_sh_op:
2204                 if (!access_ok(addr, 2))
2205                         goto sigbus;
2206
2207                 MIPS16e_compute_return_epc(regs, &oldinst);
2208                 value = regs->regs[reg];
2209                 StoreHW(addr, value, res);
2210                 if (res)
2211                         goto fault;
2212                 break;
2213
2214         case MIPS16e_sw_op:
2215         case MIPS16e_swsp_op:
2216         case MIPS16e_i8_op:     /* actually - MIPS16e_swrasp_func */
2217                 if (!access_ok(addr, 4))
2218                         goto sigbus;
2219
2220                 MIPS16e_compute_return_epc(regs, &oldinst);
2221                 value = regs->regs[reg];
2222                 StoreW(addr, value, res);
2223                 if (res)
2224                         goto fault;
2225                 break;
2226
2227         case MIPS16e_sd_op:
2228 writeDW:
2229 #ifdef CONFIG_64BIT
2230                 /*
2231                  * A 32-bit kernel might be running on a 64-bit processor.  But
2232                  * if we're on a 32-bit processor and an i-cache incoherency
2233                  * or race makes us see a 64-bit instruction here the sdl/sdr
2234                  * would blow up, so for now we don't handle unaligned 64-bit
2235                  * instructions on 32-bit kernels.
2236                  */
2237                 if (!access_ok(addr, 8))
2238                         goto sigbus;
2239
2240                 MIPS16e_compute_return_epc(regs, &oldinst);
2241                 value = regs->regs[reg];
2242                 StoreDW(addr, value, res);
2243                 if (res)
2244                         goto fault;
2245                 break;
2246 #endif /* CONFIG_64BIT */
2247
2248                 /* Cannot handle 64-bit instructions in 32-bit kernel */
2249                 goto sigill;
2250
2251         default:
2252                 /*
2253                  * Pheeee...  We encountered an yet unknown instruction or
2254                  * cache coherence problem.  Die sucker, die ...
2255                  */
2256                 goto sigill;
2257         }
2258
2259 #ifdef CONFIG_DEBUG_FS
2260         unaligned_instructions++;
2261 #endif
2262
2263         return;
2264
2265 fault:
2266         /* roll back jump/branch */
2267         regs->cp0_epc = origpc;
2268         regs->regs[31] = orig31;
2269         /* Did we have an exception handler installed? */
2270         if (fixup_exception(regs))
2271                 return;
2272
2273         die_if_kernel("Unhandled kernel unaligned access", regs);
2274         force_sig(SIGSEGV);
2275
2276         return;
2277
2278 sigbus:
2279         die_if_kernel("Unhandled kernel unaligned access", regs);
2280         force_sig(SIGBUS);
2281
2282         return;
2283
2284 sigill:
2285         die_if_kernel
2286             ("Unhandled kernel unaligned access or invalid instruction", regs);
2287         force_sig(SIGILL);
2288 }
2289
2290 asmlinkage void do_ade(struct pt_regs *regs)
2291 {
2292         enum ctx_state prev_state;
2293         unsigned int __user *pc;
2294         mm_segment_t seg;
2295
2296         prev_state = exception_enter();
2297         perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS,
2298                         1, regs, regs->cp0_badvaddr);
2299         /*
2300          * Did we catch a fault trying to load an instruction?
2301          */
2302         if (regs->cp0_badvaddr == regs->cp0_epc)
2303                 goto sigbus;
2304
2305         if (user_mode(regs) && !test_thread_flag(TIF_FIXADE))
2306                 goto sigbus;
2307         if (unaligned_action == UNALIGNED_ACTION_SIGNAL)
2308                 goto sigbus;
2309
2310         /*
2311          * Do branch emulation only if we didn't forward the exception.
2312          * This is all so but ugly ...
2313          */
2314
2315         /*
2316          * Are we running in microMIPS mode?
2317          */
2318         if (get_isa16_mode(regs->cp0_epc)) {
2319                 /*
2320                  * Did we catch a fault trying to load an instruction in
2321                  * 16-bit mode?
2322                  */
2323                 if (regs->cp0_badvaddr == msk_isa16_mode(regs->cp0_epc))
2324                         goto sigbus;
2325                 if (unaligned_action == UNALIGNED_ACTION_SHOW)
2326                         show_registers(regs);
2327
2328                 if (cpu_has_mmips) {
2329                         seg = get_fs();
2330                         if (!user_mode(regs))
2331                                 set_fs(KERNEL_DS);
2332                         emulate_load_store_microMIPS(regs,
2333                                 (void __user *)regs->cp0_badvaddr);
2334                         set_fs(seg);
2335
2336                         return;
2337                 }
2338
2339                 if (cpu_has_mips16) {
2340                         seg = get_fs();
2341                         if (!user_mode(regs))
2342                                 set_fs(KERNEL_DS);
2343                         emulate_load_store_MIPS16e(regs,
2344                                 (void __user *)regs->cp0_badvaddr);
2345                         set_fs(seg);
2346
2347                         return;
2348                 }
2349
2350                 goto sigbus;
2351         }
2352
2353         if (unaligned_action == UNALIGNED_ACTION_SHOW)
2354                 show_registers(regs);
2355         pc = (unsigned int __user *)exception_epc(regs);
2356
2357         seg = get_fs();
2358         if (!user_mode(regs))
2359                 set_fs(KERNEL_DS);
2360         emulate_load_store_insn(regs, (void __user *)regs->cp0_badvaddr, pc);
2361         set_fs(seg);
2362
2363         return;
2364
2365 sigbus:
2366         die_if_kernel("Kernel unaligned instruction access", regs);
2367         force_sig(SIGBUS);
2368
2369         /*
2370          * XXX On return from the signal handler we should advance the epc
2371          */
2372         exception_exit(prev_state);
2373 }
2374
2375 #ifdef CONFIG_DEBUG_FS
2376 static int __init debugfs_unaligned(void)
2377 {
2378         debugfs_create_u32("unaligned_instructions", S_IRUGO, mips_debugfs_dir,
2379                            &unaligned_instructions);
2380         debugfs_create_u32("unaligned_action", S_IRUGO | S_IWUSR,
2381                            mips_debugfs_dir, &unaligned_action);
2382         return 0;
2383 }
2384 arch_initcall(debugfs_unaligned);
2385 #endif