1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
2 /* Copyright (C) 2015-2018 Netronome Systems, Inc. */
4 #include <linux/delay.h>
5 #include <linux/device.h>
6 #include <linux/jiffies.h>
7 #include <linux/types.h>
8 #include <linux/slab.h>
9 #include <linux/wait.h>
12 #include "nfp6000/nfp6000.h"
14 struct nfp_cpp_mutex {
18 unsigned long long address;
22 static u32 nfp_mutex_locked(u16 interface)
24 return (u32)interface << 16 | 0x000f;
27 static u32 nfp_mutex_unlocked(u16 interface)
29 return (u32)interface << 16 | 0x0000;
32 static u32 nfp_mutex_owner(u32 val)
37 static bool nfp_mutex_is_locked(u32 val)
39 return (val & 0xffff) == 0x000f;
42 static bool nfp_mutex_is_unlocked(u32 val)
44 return (val & 0xffff) == 0000;
47 /* If you need more than 65536 recursive locks, please rethink your code. */
48 #define NFP_MUTEX_DEPTH_MAX 0xffff
51 nfp_cpp_mutex_validate(u16 interface, int *target, unsigned long long address)
53 /* Not permitted on invalid interfaces */
54 if (NFP_CPP_INTERFACE_TYPE_of(interface) ==
55 NFP_CPP_INTERFACE_TYPE_INVALID)
58 /* Address must be 64-bit aligned */
62 if (*target != NFP_CPP_TARGET_MU)
69 * nfp_cpp_mutex_init() - Initialize a mutex location
70 * @cpp: NFP CPP handle
71 * @target: NFP CPP target ID (ie NFP_CPP_TARGET_CLS or NFP_CPP_TARGET_MU)
72 * @address: Offset into the address space of the NFP CPP target ID
73 * @key: Unique 32-bit value for this mutex
75 * The CPP target:address must point to a 64-bit aligned location, and
76 * will initialize 64 bits of data at the location.
78 * This creates the initial mutex state, as locked by this
79 * nfp_cpp_interface().
81 * This function should only be called when setting up
82 * the initial lock state upon boot-up of the system.
84 * Return: 0 on success, or -errno on failure
86 int nfp_cpp_mutex_init(struct nfp_cpp *cpp,
87 int target, unsigned long long address, u32 key)
89 const u32 muw = NFP_CPP_ID(target, 4, 0); /* atomic_write */
90 u16 interface = nfp_cpp_interface(cpp);
93 err = nfp_cpp_mutex_validate(interface, &target, address);
97 err = nfp_cpp_writel(cpp, muw, address + 4, key);
101 err = nfp_cpp_writel(cpp, muw, address, nfp_mutex_locked(interface));
109 * nfp_cpp_mutex_alloc() - Create a mutex handle
110 * @cpp: NFP CPP handle
111 * @target: NFP CPP target ID (ie NFP_CPP_TARGET_CLS or NFP_CPP_TARGET_MU)
112 * @address: Offset into the address space of the NFP CPP target ID
113 * @key: 32-bit unique key (must match the key at this location)
115 * The CPP target:address must point to a 64-bit aligned location, and
116 * reserve 64 bits of data at the location for use by the handle.
118 * Only target/address pairs that point to entities that support the
119 * MU Atomic Engine's CmpAndSwap32 command are supported.
121 * Return: A non-NULL struct nfp_cpp_mutex * on success, NULL on failure.
123 struct nfp_cpp_mutex *nfp_cpp_mutex_alloc(struct nfp_cpp *cpp, int target,
124 unsigned long long address, u32 key)
126 const u32 mur = NFP_CPP_ID(target, 3, 0); /* atomic_read */
127 u16 interface = nfp_cpp_interface(cpp);
128 struct nfp_cpp_mutex *mutex;
132 err = nfp_cpp_mutex_validate(interface, &target, address);
136 err = nfp_cpp_readl(cpp, mur, address + 4, &tmp);
143 mutex = kzalloc(sizeof(*mutex), GFP_KERNEL);
148 mutex->target = target;
149 mutex->address = address;
157 * nfp_cpp_mutex_free() - Free a mutex handle - does not alter the lock state
158 * @mutex: NFP CPP Mutex handle
160 void nfp_cpp_mutex_free(struct nfp_cpp_mutex *mutex)
166 * nfp_cpp_mutex_lock() - Lock a mutex handle, using the NFP MU Atomic Engine
167 * @mutex: NFP CPP Mutex handle
169 * Return: 0 on success, or -errno on failure
171 int nfp_cpp_mutex_lock(struct nfp_cpp_mutex *mutex)
173 unsigned long warn_at = jiffies + NFP_MUTEX_WAIT_FIRST_WARN * HZ;
174 unsigned long err_at = jiffies + NFP_MUTEX_WAIT_ERROR * HZ;
175 unsigned int timeout_ms = 1;
178 /* We can't use a waitqueue here, because the unlocker
179 * might be on a separate CPU.
181 * So just wait for now.
184 err = nfp_cpp_mutex_trylock(mutex);
188 err = msleep_interruptible(timeout_ms);
191 "interrupted waiting for NFP mutex\n");
195 if (time_is_before_eq_jiffies(warn_at)) {
196 warn_at = jiffies + NFP_MUTEX_WAIT_NEXT_WARN * HZ;
198 "Warning: waiting for NFP mutex [depth:%hd target:%d addr:%llx key:%08x]\n",
200 mutex->target, mutex->address, mutex->key);
202 if (time_is_before_eq_jiffies(err_at)) {
203 nfp_err(mutex->cpp, "Error: mutex wait timed out\n");
212 * nfp_cpp_mutex_unlock() - Unlock a mutex handle, using the MU Atomic Engine
213 * @mutex: NFP CPP Mutex handle
215 * Return: 0 on success, or -errno on failure
217 int nfp_cpp_mutex_unlock(struct nfp_cpp_mutex *mutex)
219 const u32 muw = NFP_CPP_ID(mutex->target, 4, 0); /* atomic_write */
220 const u32 mur = NFP_CPP_ID(mutex->target, 3, 0); /* atomic_read */
221 struct nfp_cpp *cpp = mutex->cpp;
226 interface = nfp_cpp_interface(cpp);
228 if (mutex->depth > 1) {
233 err = nfp_cpp_readl(mutex->cpp, mur, mutex->address + 4, &key);
237 if (key != mutex->key)
240 err = nfp_cpp_readl(mutex->cpp, mur, mutex->address, &value);
244 if (value != nfp_mutex_locked(interface))
247 err = nfp_cpp_writel(cpp, muw, mutex->address,
248 nfp_mutex_unlocked(interface));
257 * nfp_cpp_mutex_trylock() - Attempt to lock a mutex handle
258 * @mutex: NFP CPP Mutex handle
260 * Return: 0 if the lock succeeded, -errno on failure
262 int nfp_cpp_mutex_trylock(struct nfp_cpp_mutex *mutex)
264 const u32 muw = NFP_CPP_ID(mutex->target, 4, 0); /* atomic_write */
265 const u32 mus = NFP_CPP_ID(mutex->target, 5, 3); /* test_set_imm */
266 const u32 mur = NFP_CPP_ID(mutex->target, 3, 0); /* atomic_read */
267 struct nfp_cpp *cpp = mutex->cpp;
271 if (mutex->depth > 0) {
272 if (mutex->depth == NFP_MUTEX_DEPTH_MAX)
278 /* Verify that the lock marker is not damaged */
279 err = nfp_cpp_readl(cpp, mur, mutex->address + 4, &key);
283 if (key != mutex->key)
286 /* Compare against the unlocked state, and if true,
287 * write the interface id into the top 16 bits, and
290 value = nfp_mutex_locked(nfp_cpp_interface(cpp));
292 /* We use test_set_imm here, as it implies a read
293 * of the current state, and sets the bits in the
294 * bytemask of the command to 1s. Since the mutex
295 * is guaranteed to be 64-bit aligned, the bytemask
296 * of this 32-bit command is ensured to be 8'b00001111,
297 * which implies that the lower 4 bits will be set to
298 * ones regardless of the initial state.
300 * Since this is a 'Readback' operation, with no Pull
301 * data, we can treat this as a normal Push (read)
302 * atomic, which returns the original value.
304 err = nfp_cpp_readl(cpp, mus, mutex->address, &tmp);
308 /* Was it unlocked? */
309 if (nfp_mutex_is_unlocked(tmp)) {
310 /* The read value can only be 0x....0000 in the unlocked state.
311 * If there was another contending for this lock, then
312 * the lock state would be 0x....000f
315 /* Write our owner ID into the lock
316 * While not strictly necessary, this helps with
317 * debug and bookkeeping.
319 err = nfp_cpp_writel(cpp, muw, mutex->address, value);
327 return nfp_mutex_is_locked(tmp) ? -EBUSY : -EINVAL;
331 * nfp_cpp_mutex_reclaim() - Unlock mutex if held by local endpoint
332 * @cpp: NFP CPP handle
333 * @target: NFP CPP target ID (ie NFP_CPP_TARGET_CLS or NFP_CPP_TARGET_MU)
334 * @address: Offset into the address space of the NFP CPP target ID
336 * Release lock if held by local system. Extreme care is advised, call only
337 * when no local lock users can exist.
339 * Return: 0 if the lock was OK, 1 if locked by us, -errno on invalid mutex
341 int nfp_cpp_mutex_reclaim(struct nfp_cpp *cpp, int target,
342 unsigned long long address)
344 const u32 mur = NFP_CPP_ID(target, 3, 0); /* atomic_read */
345 const u32 muw = NFP_CPP_ID(target, 4, 0); /* atomic_write */
346 u16 interface = nfp_cpp_interface(cpp);
350 err = nfp_cpp_mutex_validate(interface, &target, address);
355 err = nfp_cpp_readl(cpp, mur, address, &tmp);
359 if (nfp_mutex_is_unlocked(tmp) || nfp_mutex_owner(tmp) != interface)
363 err = nfp_cpp_writel(cpp, muw, address, nfp_mutex_unlocked(interface));