arch/arm/lib/memcpy.S

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 /*
   3  *  linux/arch/arm/lib/memcpy.S
   4  *
   5  *  Author:     Nicolas Pitre
   6  *  Created:    Sep 28, 2005
   7  *  Copyright:  MontaVista Software, Inc.
   8  */
   9
  10 #include <linux/linkage.h>
  11 #include <asm/assembler.h>
  12
  13 #define LDR1W_SHIFT     0
  14 #define STR1W_SHIFT     0
  15
  16         .macro ldr1w ptr reg abort
  17         W(ldr) \reg, [\ptr], #4
  18         .endm
  19
  20         .macro ldr4w ptr reg1 reg2 reg3 reg4 abort
  21         ldmia \ptr!, {\reg1, \reg2, \reg3, \reg4}
  22         .endm
  23
  24         .macro ldr8w ptr reg1 reg2 reg3 reg4 reg5 reg6 reg7 reg8 abort
  25         ldmia \ptr!, {\reg1, \reg2, \reg3, \reg4, \reg5, \reg6, \reg7, \reg8}
  26         .endm
  27
  28         .macro ldr1b ptr reg cond=al abort
  29         ldrb\cond\() \reg, [\ptr], #1
  30         .endm
  31
  32         .macro str1w ptr reg abort
  33         W(str) \reg, [\ptr], #4
  34         .endm
  35
  36         .macro str8w ptr reg1 reg2 reg3 reg4 reg5 reg6 reg7 reg8 abort
  37         stmia \ptr!, {\reg1, \reg2, \reg3, \reg4, \reg5, \reg6, \reg7, \reg8}
  38         .endm
  39
  40         .macro str1b ptr reg cond=al abort
  41         strb\cond\() \reg, [\ptr], #1
  42         .endm
  43
  44         .macro enter reg1 reg2
  45         stmdb sp!, {r0, \reg1, \reg2}
  46         .endm
  47
  48         .macro exit reg1 reg2
  49         ldmfd sp!, {r0, \reg1, \reg2}
  50         .endm
  51
  52         .text
  53
  54 /* Prototype: void *memcpy(void *dest, const void *src, size_t n); */
  55         .syntax unified
  56 #if CONFIG_IS_ENABLED(SYS_THUMB_BUILD) && !defined(MEMCPY_NO_THUMB_BUILD)
  57         .thumb
  58         .thumb_func
  59 #endif
  60 ENTRY(memcpy)
  61                 cmp     r0, r1
  62                 bxeq    lr
  63
  64                 enter   r4, lr
  65
  66                 subs    r2, r2, #4
  67                 blt     8f
  68                 ands    ip, r0, #3
  69         PLD(    pld     [r1, #0]                )
  70                 bne     9f
  71                 ands    ip, r1, #3
  72                 bne     10f
  73
  74 1:              subs    r2, r2, #(28)
  75                 stmfd   sp!, {r5 - r8}
  76                 blt     5f
  77
  78         CALGN(  ands    ip, r0, #31             )
  79         CALGN(  rsb     r3, ip, #32             )
  80         CALGN(  sbcsne  r4, r3, r2              )  @ C is always set here
  81         CALGN(  bcs     2f                      )
  82         CALGN(  adr     r4, 6f                  )
  83         CALGN(  subs    r2, r2, r3              )  @ C gets set
  84         CALGN(  add     pc, r4, ip              )
  85
  86         PLD(    pld     [r1, #0]                )
  87 2:      PLD(    subs    r2, r2, #96             )
  88         PLD(    pld     [r1, #28]               )
  89         PLD(    blt     4f                      )
  90         PLD(    pld     [r1, #60]               )
  91         PLD(    pld     [r1, #92]               )
  92
  93 3:      PLD(    pld     [r1, #124]              )
  94 4:              ldr8w   r1, r3, r4, r5, r6, r7, r8, ip, lr, abort=20f
  95                 subs    r2, r2, #32
  96                 str8w   r0, r3, r4, r5, r6, r7, r8, ip, lr, abort=20f
  97                 bge     3b
  98         PLD(    cmn     r2, #96                 )
  99         PLD(    bge     4b                      )
 100
 101 5:              ands    ip, r2, #28
 102                 rsb     ip, ip, #32
 103 #if LDR1W_SHIFT > 0
 104                 lsl     ip, ip, #LDR1W_SHIFT
 105 #endif
 106                 addne   pc, pc, ip              @ C is always clear here
 107                 b       7f
 108 6:
 109                 .rept   (1 << LDR1W_SHIFT)
 110                 W(nop)
 111                 .endr
 112                 ldr1w   r1, r3, abort=20f
 113                 ldr1w   r1, r4, abort=20f
 114                 ldr1w   r1, r5, abort=20f
 115                 ldr1w   r1, r6, abort=20f
 116                 ldr1w   r1, r7, abort=20f
 117                 ldr1w   r1, r8, abort=20f
 118                 ldr1w   r1, lr, abort=20f
 119
 120 #if LDR1W_SHIFT < STR1W_SHIFT
 121                 lsl     ip, ip, #STR1W_SHIFT - LDR1W_SHIFT
 122 #elif LDR1W_SHIFT > STR1W_SHIFT
 123                 lsr     ip, ip, #LDR1W_SHIFT - STR1W_SHIFT
 124 #endif
 125                 add     pc, pc, ip
 126                 nop
 127                 .rept   (1 << STR1W_SHIFT)
 128                 W(nop)
 129                 .endr
 130                 str1w   r0, r3, abort=20f
 131                 str1w   r0, r4, abort=20f
 132                 str1w   r0, r5, abort=20f
 133                 str1w   r0, r6, abort=20f
 134                 str1w   r0, r7, abort=20f
 135                 str1w   r0, r8, abort=20f
 136                 str1w   r0, lr, abort=20f
 137
 138         CALGN(  bcs     2b                      )
 139
 140 7:              ldmfd   sp!, {r5 - r8}
 141
 142 8:              movs    r2, r2, lsl #31
 143                 ldr1b   r1, r3, ne, abort=21f
 144                 ldr1b   r1, r4, cs, abort=21f
 145                 ldr1b   r1, ip, cs, abort=21f
 146                 str1b   r0, r3, ne, abort=21f
 147                 str1b   r0, r4, cs, abort=21f
 148                 str1b   r0, ip, cs, abort=21f
 149
 150                 exit    r4, lr
 151                 bx      lr
 152
 153 9:              rsb     ip, ip, #4
 154                 cmp     ip, #2
 155                 ldr1b   r1, r3, gt, abort=21f
 156                 ldr1b   r1, r4, ge, abort=21f
 157                 ldr1b   r1, lr, abort=21f
 158                 str1b   r0, r3, gt, abort=21f
 159                 str1b   r0, r4, ge, abort=21f
 160                 subs    r2, r2, ip
 161                 str1b   r0, lr, abort=21f
 162                 blt     8b
 163                 ands    ip, r1, #3
 164                 beq     1b
 165
 166 10:             bic     r1, r1, #3
 167                 cmp     ip, #2
 168                 ldr1w   r1, lr, abort=21f
 169                 beq     17f
 170                 bgt     18f
 171
 172
 173                 .macro  forward_copy_shift pull push
 174
 175                 subs    r2, r2, #28
 176                 blt     14f
 177
 178         CALGN(  ands    ip, r0, #31             )
 179         CALGN(  rsb     ip, ip, #32             )
 180         CALGN(  sbcsne  r4, ip, r2              )  @ C is always set here
 181         CALGN(  subcc   r2, r2, ip              )
 182         CALGN(  bcc     15f                     )
 183
 184 11:             stmfd   sp!, {r5 - r9}
 185
 186         PLD(    pld     [r1, #0]                )
 187         PLD(    subs    r2, r2, #96             )
 188         PLD(    pld     [r1, #28]               )
 189         PLD(    blt     13f                     )
 190         PLD(    pld     [r1, #60]               )
 191         PLD(    pld     [r1, #92]               )
 192
 193 12:     PLD(    pld     [r1, #124]              )
 194 13:             ldr4w   r1, r4, r5, r6, r7, abort=19f
 195                 mov     r3, lr, lspull #\pull
 196                 subs    r2, r2, #32
 197                 ldr4w   r1, r8, r9, ip, lr, abort=19f
 198                 orr     r3, r3, r4, lspush #\push
 199                 mov     r4, r4, lspull #\pull
 200                 orr     r4, r4, r5, lspush #\push
 201                 mov     r5, r5, lspull #\pull
 202                 orr     r5, r5, r6, lspush #\push
 203                 mov     r6, r6, lspull #\pull
 204                 orr     r6, r6, r7, lspush #\push
 205                 mov     r7, r7, lspull #\pull
 206                 orr     r7, r7, r8, lspush #\push
 207                 mov     r8, r8, lspull #\pull
 208                 orr     r8, r8, r9, lspush #\push
 209                 mov     r9, r9, lspull #\pull
 210                 orr     r9, r9, ip, lspush #\push
 211                 mov     ip, ip, lspull #\pull
 212                 orr     ip, ip, lr, lspush #\push
 213                 str8w   r0, r3, r4, r5, r6, r7, r8, r9, ip, , abort=19f
 214                 bge     12b
 215         PLD(    cmn     r2, #96                 )
 216         PLD(    bge     13b                     )
 217
 218                 ldmfd   sp!, {r5 - r9}
 219
 220 14:             ands    ip, r2, #28
 221                 beq     16f
 222
 223 15:             mov     r3, lr, lspull #\pull
 224                 ldr1w   r1, lr, abort=21f
 225                 subs    ip, ip, #4
 226                 orr     r3, r3, lr, lspush #\push
 227                 str1w   r0, r3, abort=21f
 228                 bgt     15b
 229         CALGN(  cmp     r2, #0                  )
 230         CALGN(  bge     11b                     )
 231
 232 16:             sub     r1, r1, #(\push / 8)
 233                 b       8b
 234
 235                 .endm
 236
 237
 238                 forward_copy_shift      pull=8  push=24
 239
 240 17:             forward_copy_shift      pull=16 push=16
 241
 242 18:             forward_copy_shift      pull=24 push=8
 243
 244
 245 /*
 246  * Abort preamble and completion macros.
 247  * If a fixup handler is required then those macros must surround it.
 248  * It is assumed that the fixup code will handle the private part of
 249  * the exit macro.
 250  */
 251
 252         .macro  copy_abort_preamble
 253 19:     ldmfd   sp!, {r5 - r9}
 254         b       21f
 255 20:     ldmfd   sp!, {r5 - r8}
 256 21:
 257         .endm
 258
 259         .macro  copy_abort_end
 260         ldmfd   sp!, {r4, lr}
 261         bx      lr
 262         .endm
 263
 264 ENDPROC(memcpy)