arch/arm64/include/asm/atomic_ll_sc.h

   1 /* SPDX-License-Identifier: GPL-2.0-only */
   2 /*
   3  * Based on arch/arm/include/asm/atomic.h
   4  *
   5  * Copyright (C) 1996 Russell King.
   6  * Copyright (C) 2002 Deep Blue Solutions Ltd.
   7  * Copyright (C) 2012 ARM Ltd.
   8  */
   9
  10 #ifndef __ASM_ATOMIC_LL_SC_H
  11 #define __ASM_ATOMIC_LL_SC_H
  12
  13 #ifndef __ARM64_IN_ATOMIC_IMPL
  14 #error "please don't include this file directly"
  15 #endif
  16
  17 /*
  18  * AArch64 UP and SMP safe atomic ops.  We use load exclusive and
  19  * store exclusive to ensure that these are atomic.  We may loop
  20  * to ensure that the update happens.
  21  *
  22  * NOTE: these functions do *not* follow the PCS and must explicitly
  23  * save any clobbered registers other than x0 (regardless of return
  24  * value).  This is achieved through -fcall-saved-* compiler flags for
  25  * this file, which unfortunately don't work on a per-function basis
  26  * (the optimize attribute silently ignores these options).
  27  */
  28
  29 #define ATOMIC_OP(op, asm_op, constraint)                               \
  30 __LL_SC_INLINE void                                                     \
  31 __LL_SC_PREFIX(arch_atomic_##op(int i, atomic_t *v))                    \
  32 {                                                                       \
  33         unsigned long tmp;                                              \
  34         int result;                                                     \
  35                                                                         \
  36         asm volatile("// atomic_" #op "\n"                              \
  37 "       prfm    pstl1strm, %2\n"                                        \
  38 "1:     ldxr    %w0, %2\n"                                              \
  39 "       " #asm_op "     %w0, %w0, %w3\n"                                \
  40 "       stxr    %w1, %w0, %2\n"                                         \
  41 "       cbnz    %w1, 1b"                                                \
  42         : "=&r" (result), "=&r" (tmp), "+Q" (v->counter)                \
  43         : #constraint "r" (i));                                         \
  44 }                                                                       \
  45 __LL_SC_EXPORT(arch_atomic_##op);
  46
  47 #define ATOMIC_OP_RETURN(name, mb, acq, rel, cl, op, asm_op, constraint)\
  48 __LL_SC_INLINE int                                                      \
  49 __LL_SC_PREFIX(arch_atomic_##op##_return##name(int i, atomic_t *v))     \
  50 {                                                                       \
  51         unsigned long tmp;                                              \
  52         int result;                                                     \
  53                                                                         \
  54         asm volatile("// atomic_" #op "_return" #name "\n"              \
  55 "       prfm    pstl1strm, %2\n"                                        \
  56 "1:     ld" #acq "xr    %w0, %2\n"                                      \
  57 "       " #asm_op "     %w0, %w0, %w3\n"                                \
  58 "       st" #rel "xr    %w1, %w0, %2\n"                                 \
  59 "       cbnz    %w1, 1b\n"                                              \
  60 "       " #mb                                                           \
  61         : "=&r" (result), "=&r" (tmp), "+Q" (v->counter)                \
  62         : #constraint "r" (i)                                           \
  63         : cl);                                                          \
  64                                                                         \
  65         return result;                                                  \
  66 }                                                                       \
  67 __LL_SC_EXPORT(arch_atomic_##op##_return##name);
  68
  69 #define ATOMIC_FETCH_OP(name, mb, acq, rel, cl, op, asm_op, constraint) \
  70 __LL_SC_INLINE int                                                      \
  71 __LL_SC_PREFIX(arch_atomic_fetch_##op##name(int i, atomic_t *v))        \
  72 {                                                                       \
  73         unsigned long tmp;                                              \
  74         int val, result;                                                \
  75                                                                         \
  76         asm volatile("// atomic_fetch_" #op #name "\n"                  \
  77 "       prfm    pstl1strm, %3\n"                                        \
  78 "1:     ld" #acq "xr    %w0, %3\n"                                      \
  79 "       " #asm_op "     %w1, %w0, %w4\n"                                \
  80 "       st" #rel "xr    %w2, %w1, %3\n"                                 \
  81 "       cbnz    %w2, 1b\n"                                              \
  82 "       " #mb                                                           \
  83         : "=&r" (result), "=&r" (val), "=&r" (tmp), "+Q" (v->counter)   \
  84         : #constraint "r" (i)                                           \
  85         : cl);                                                          \
  86                                                                         \
  87         return result;                                                  \
  88 }                                                                       \
  89 __LL_SC_EXPORT(arch_atomic_fetch_##op##name);
  90
  91 #define ATOMIC_OPS(...)                                                 \
  92         ATOMIC_OP(__VA_ARGS__)                                          \
  93         ATOMIC_OP_RETURN(        , dmb ish,  , l, "memory", __VA_ARGS__)\
  94         ATOMIC_OP_RETURN(_relaxed,        ,  ,  ,         , __VA_ARGS__)\
  95         ATOMIC_OP_RETURN(_acquire,        , a,  , "memory", __VA_ARGS__)\
  96         ATOMIC_OP_RETURN(_release,        ,  , l, "memory", __VA_ARGS__)\
  97         ATOMIC_FETCH_OP (        , dmb ish,  , l, "memory", __VA_ARGS__)\
  98         ATOMIC_FETCH_OP (_relaxed,        ,  ,  ,         , __VA_ARGS__)\
  99         ATOMIC_FETCH_OP (_acquire,        , a,  , "memory", __VA_ARGS__)\
 100         ATOMIC_FETCH_OP (_release,        ,  , l, "memory", __VA_ARGS__)
 101
 102 ATOMIC_OPS(add, add, I)
 103 ATOMIC_OPS(sub, sub, J)
 104
 105 #undef ATOMIC_OPS
 106 #define ATOMIC_OPS(...)                                                 \
 107         ATOMIC_OP(__VA_ARGS__)                                          \
 108         ATOMIC_FETCH_OP (        , dmb ish,  , l, "memory", __VA_ARGS__)\
 109         ATOMIC_FETCH_OP (_relaxed,        ,  ,  ,         , __VA_ARGS__)\
 110         ATOMIC_FETCH_OP (_acquire,        , a,  , "memory", __VA_ARGS__)\
 111         ATOMIC_FETCH_OP (_release,        ,  , l, "memory", __VA_ARGS__)
 112
 113 ATOMIC_OPS(and, and, )
 114 ATOMIC_OPS(andnot, bic, )
 115 ATOMIC_OPS(or, orr, )
 116 ATOMIC_OPS(xor, eor, )
 117
 118 #undef ATOMIC_OPS
 119 #undef ATOMIC_FETCH_OP
 120 #undef ATOMIC_OP_RETURN
 121 #undef ATOMIC_OP
 122
 123 #define ATOMIC64_OP(op, asm_op, constraint)                             \
 124 __LL_SC_INLINE void                                                     \
 125 __LL_SC_PREFIX(arch_atomic64_##op(s64 i, atomic64_t *v))                \
 126 {                                                                       \
 127         s64 result;                                                     \
 128         unsigned long tmp;                                              \
 129                                                                         \
 130         asm volatile("// atomic64_" #op "\n"                            \
 131 "       prfm    pstl1strm, %2\n"                                        \
 132 "1:     ldxr    %0, %2\n"                                               \
 133 "       " #asm_op "     %0, %0, %3\n"                                   \
 134 "       stxr    %w1, %0, %2\n"                                          \
 135 "       cbnz    %w1, 1b"                                                \
 136         : "=&r" (result), "=&r" (tmp), "+Q" (v->counter)                \
 137         : #constraint "r" (i));                                         \
 138 }                                                                       \
 139 __LL_SC_EXPORT(arch_atomic64_##op);
 140
 141 #define ATOMIC64_OP_RETURN(name, mb, acq, rel, cl, op, asm_op, constraint)\
 142 __LL_SC_INLINE s64                                                      \
 143 __LL_SC_PREFIX(arch_atomic64_##op##_return##name(s64 i, atomic64_t *v))\
 144 {                                                                       \
 145         s64 result;                                                     \
 146         unsigned long tmp;                                              \
 147                                                                         \
 148         asm volatile("// atomic64_" #op "_return" #name "\n"            \
 149 "       prfm    pstl1strm, %2\n"                                        \
 150 "1:     ld" #acq "xr    %0, %2\n"                                       \
 151 "       " #asm_op "     %0, %0, %3\n"                                   \
 152 "       st" #rel "xr    %w1, %0, %2\n"                                  \
 153 "       cbnz    %w1, 1b\n"                                              \
 154 "       " #mb                                                           \
 155         : "=&r" (result), "=&r" (tmp), "+Q" (v->counter)                \
 156         : #constraint "r" (i)                                           \
 157         : cl);                                                          \
 158                                                                         \
 159         return result;                                                  \
 160 }                                                                       \
 161 __LL_SC_EXPORT(arch_atomic64_##op##_return##name);
 162
 163 #define ATOMIC64_FETCH_OP(name, mb, acq, rel, cl, op, asm_op, constraint)\
 164 __LL_SC_INLINE s64                                                      \
 165 __LL_SC_PREFIX(arch_atomic64_fetch_##op##name(s64 i, atomic64_t *v))    \
 166 {                                                                       \
 167         s64 result, val;                                                \
 168         unsigned long tmp;                                              \
 169                                                                         \
 170         asm volatile("// atomic64_fetch_" #op #name "\n"                \
 171 "       prfm    pstl1strm, %3\n"                                        \
 172 "1:     ld" #acq "xr    %0, %3\n"                                       \
 173 "       " #asm_op "     %1, %0, %4\n"                                   \
 174 "       st" #rel "xr    %w2, %1, %3\n"                                  \
 175 "       cbnz    %w2, 1b\n"                                              \
 176 "       " #mb                                                           \
 177         : "=&r" (result), "=&r" (val), "=&r" (tmp), "+Q" (v->counter)   \
 178         : #constraint "r" (i)                                           \
 179         : cl);                                                          \
 180                                                                         \
 181         return result;                                                  \
 182 }                                                                       \
 183 __LL_SC_EXPORT(arch_atomic64_fetch_##op##name);
 184
 185 #define ATOMIC64_OPS(...)                                               \
 186         ATOMIC64_OP(__VA_ARGS__)                                        \
 187         ATOMIC64_OP_RETURN(, dmb ish,  , l, "memory", __VA_ARGS__)      \
 188         ATOMIC64_OP_RETURN(_relaxed,,  ,  ,         , __VA_ARGS__)      \
 189         ATOMIC64_OP_RETURN(_acquire,, a,  , "memory", __VA_ARGS__)      \
 190         ATOMIC64_OP_RETURN(_release,,  , l, "memory", __VA_ARGS__)      \
 191         ATOMIC64_FETCH_OP (, dmb ish,  , l, "memory", __VA_ARGS__)      \
 192         ATOMIC64_FETCH_OP (_relaxed,,  ,  ,         , __VA_ARGS__)      \
 193         ATOMIC64_FETCH_OP (_acquire,, a,  , "memory", __VA_ARGS__)      \
 194         ATOMIC64_FETCH_OP (_release,,  , l, "memory", __VA_ARGS__)
 195
 196 ATOMIC64_OPS(add, add, I)
 197 ATOMIC64_OPS(sub, sub, J)
 198
 199 #undef ATOMIC64_OPS
 200 #define ATOMIC64_OPS(...)                                               \
 201         ATOMIC64_OP(__VA_ARGS__)                                        \
 202         ATOMIC64_FETCH_OP (, dmb ish,  , l, "memory", __VA_ARGS__)      \
 203         ATOMIC64_FETCH_OP (_relaxed,,  ,  ,         , __VA_ARGS__)      \
 204         ATOMIC64_FETCH_OP (_acquire,, a,  , "memory", __VA_ARGS__)      \
 205         ATOMIC64_FETCH_OP (_release,,  , l, "memory", __VA_ARGS__)
 206
 207 ATOMIC64_OPS(and, and, L)
 208 ATOMIC64_OPS(andnot, bic, )
 209 ATOMIC64_OPS(or, orr, L)
 210 ATOMIC64_OPS(xor, eor, L)
 211
 212 #undef ATOMIC64_OPS
 213 #undef ATOMIC64_FETCH_OP
 214 #undef ATOMIC64_OP_RETURN
 215 #undef ATOMIC64_OP
 216
 217 __LL_SC_INLINE s64
 218 __LL_SC_PREFIX(arch_atomic64_dec_if_positive(atomic64_t *v))
 219 {
 220         s64 result;
 221         unsigned long tmp;
 222
 223         asm volatile("// atomic64_dec_if_positive\n"
 224 "       prfm    pstl1strm, %2\n"
 225 "1:     ldxr    %0, %2\n"
 226 "       subs    %0, %0, #1\n"
 227 "       b.lt    2f\n"
 228 "       stlxr   %w1, %0, %2\n"
 229 "       cbnz    %w1, 1b\n"
 230 "       dmb     ish\n"
 231 "2:"
 232         : "=&r" (result), "=&r" (tmp), "+Q" (v->counter)
 233         :
 234         : "cc", "memory");
 235
 236         return result;
 237 }
 238 __LL_SC_EXPORT(arch_atomic64_dec_if_positive);
 239
 240 #define __CMPXCHG_CASE(w, sfx, name, sz, mb, acq, rel, cl, constraint)  \
 241 __LL_SC_INLINE u##sz                                                    \
 242 __LL_SC_PREFIX(__cmpxchg_case_##name##sz(volatile void *ptr,            \
 243                                          unsigned long old,             \
 244                                          u##sz new))                    \
 245 {                                                                       \
 246         unsigned long tmp;                                              \
 247         u##sz oldval;                                                   \
 248                                                                         \
 249         /*                                                              \
 250          * Sub-word sizes require explicit casting so that the compare  \
 251          * part of the cmpxchg doesn't end up interpreting non-zero     \
 252          * upper bits of the register containing "old".                 \
 253          */                                                             \
 254         if (sz < 32)                                                    \
 255                 old = (u##sz)old;                                       \
 256                                                                         \
 257         asm volatile(                                                   \
 258         "       prfm    pstl1strm, %[v]\n"                              \
 259         "1:     ld" #acq "xr" #sfx "\t%" #w "[oldval], %[v]\n"          \
 260         "       eor     %" #w "[tmp], %" #w "[oldval], %" #w "[old]\n"  \
 261         "       cbnz    %" #w "[tmp], 2f\n"                             \
 262         "       st" #rel "xr" #sfx "\t%w[tmp], %" #w "[new], %[v]\n"    \
 263         "       cbnz    %w[tmp], 1b\n"                                  \
 264         "       " #mb "\n"                                              \
 265         "2:"                                                            \
 266         : [tmp] "=&r" (tmp), [oldval] "=&r" (oldval),                   \
 267           [v] "+Q" (*(u##sz *)ptr)                                      \
 268         : [old] #constraint "r" (old), [new] "r" (new)                  \
 269         : cl);                                                          \
 270                                                                         \
 271         return oldval;                                                  \
 272 }                                                                       \
 273 __LL_SC_EXPORT(__cmpxchg_case_##name##sz);
 274
 275 /*
 276  * Earlier versions of GCC (no later than 8.1.0) appear to incorrectly
 277  * handle the 'K' constraint for the value 4294967295 - thus we use no
 278  * constraint for 32 bit operations.
 279  */
 280 __CMPXCHG_CASE(w, b,     ,  8,        ,  ,  ,         , )
 281 __CMPXCHG_CASE(w, h,     , 16,        ,  ,  ,         , )
 282 __CMPXCHG_CASE(w,  ,     , 32,        ,  ,  ,         , )
 283 __CMPXCHG_CASE( ,  ,     , 64,        ,  ,  ,         , L)
 284 __CMPXCHG_CASE(w, b, acq_,  8,        , a,  , "memory", )
 285 __CMPXCHG_CASE(w, h, acq_, 16,        , a,  , "memory", )
 286 __CMPXCHG_CASE(w,  , acq_, 32,        , a,  , "memory", )
 287 __CMPXCHG_CASE( ,  , acq_, 64,        , a,  , "memory", L)
 288 __CMPXCHG_CASE(w, b, rel_,  8,        ,  , l, "memory", )
 289 __CMPXCHG_CASE(w, h, rel_, 16,        ,  , l, "memory", )
 290 __CMPXCHG_CASE(w,  , rel_, 32,        ,  , l, "memory", )
 291 __CMPXCHG_CASE( ,  , rel_, 64,        ,  , l, "memory", L)
 292 __CMPXCHG_CASE(w, b,  mb_,  8, dmb ish,  , l, "memory", )
 293 __CMPXCHG_CASE(w, h,  mb_, 16, dmb ish,  , l, "memory", )
 294 __CMPXCHG_CASE(w,  ,  mb_, 32, dmb ish,  , l, "memory", )
 295 __CMPXCHG_CASE( ,  ,  mb_, 64, dmb ish,  , l, "memory", L)
 296
 297 #undef __CMPXCHG_CASE
 298
 299 #define __CMPXCHG_DBL(name, mb, rel, cl)                                \
 300 __LL_SC_INLINE long                                                     \
 301 __LL_SC_PREFIX(__cmpxchg_double##name(unsigned long old1,               \
 302                                       unsigned long old2,               \
 303                                       unsigned long new1,               \
 304                                       unsigned long new2,               \
 305                                       volatile void *ptr))              \
 306 {                                                                       \
 307         unsigned long tmp, ret;                                         \
 308                                                                         \
 309         asm volatile("// __cmpxchg_double" #name "\n"                   \
 310         "       prfm    pstl1strm, %2\n"                                \
 311         "1:     ldxp    %0, %1, %2\n"                                   \
 312         "       eor     %0, %0, %3\n"                                   \
 313         "       eor     %1, %1, %4\n"                                   \
 314         "       orr     %1, %0, %1\n"                                   \
 315         "       cbnz    %1, 2f\n"                                       \
 316         "       st" #rel "xp    %w0, %5, %6, %2\n"                      \
 317         "       cbnz    %w0, 1b\n"                                      \
 318         "       " #mb "\n"                                              \
 319         "2:"                                                            \
 320         : "=&r" (tmp), "=&r" (ret), "+Q" (*(unsigned long *)ptr)        \
 321         : "r" (old1), "r" (old2), "r" (new1), "r" (new2)                \
 322         : cl);                                                          \
 323                                                                         \
 324         return ret;                                                     \
 325 }                                                                       \
 326 __LL_SC_EXPORT(__cmpxchg_double##name);
 327
 328 __CMPXCHG_DBL(   ,        ,  ,         )
 329 __CMPXCHG_DBL(_mb, dmb ish, l, "memory")
 330
 331 #undef __CMPXCHG_DBL
 332
 333 #endif  /* __ASM_ATOMIC_LL_SC_H */