1 // SPDX-License-Identifier: GPL-2.0
3 * Copyright (c) 2016, NVIDIA CORPORATION.
8 #include <asm/arch-tegra/ivc.h>
10 #define TEGRA_IVC_ALIGN 64
13 * IVC channel reset protocol.
15 * Each end uses its tx_channel.state to indicate its synchronization state.
19 * This value is zero for backwards compatibility with services that
20 * assume channels to be initially zeroed. Such channels are in an
21 * initially valid state, but cannot be asynchronously reset, and must
22 * maintain a valid state at all times.
24 * The transmitting end can enter the established state from the sync or
25 * ack state when it observes the receiving endpoint in the ack or
26 * established state, indicating that has cleared the counters in our
29 ivc_state_established = 0,
32 * If an endpoint is observed in the sync state, the remote endpoint is
33 * allowed to clear the counters it owns asynchronously with respect to
34 * the current endpoint. Therefore, the current endpoint is no longer
35 * allowed to communicate.
40 * When the transmitting end observes the receiving end in the sync
41 * state, it can clear the w_count and r_count and transition to the ack
42 * state. If the remote endpoint observes us in the ack state, it can
43 * return to the established state once it has cleared its counters.
49 * This structure is divided into two-cache aligned parts, the first is only
50 * written through the tx_channel pointer, while the second is only written
51 * through the rx_channel pointer. This delineates ownership of the cache lines,
52 * which is critical to performance and necessary in non-cache coherent
55 struct tegra_ivc_channel_header {
57 /* fields owned by the transmitting end */
62 uint8_t w_align[TEGRA_IVC_ALIGN];
65 /* fields owned by the receiving end */
67 uint8_t r_align[TEGRA_IVC_ALIGN];
71 static inline void tegra_ivc_invalidate_counter(struct tegra_ivc *ivc,
72 struct tegra_ivc_channel_header *h,
75 ulong base = ((ulong)h) + offset;
76 invalidate_dcache_range(base, base + TEGRA_IVC_ALIGN);
79 static inline void tegra_ivc_flush_counter(struct tegra_ivc *ivc,
80 struct tegra_ivc_channel_header *h,
83 ulong base = ((ulong)h) + offset;
84 flush_dcache_range(base, base + TEGRA_IVC_ALIGN);
87 static inline ulong tegra_ivc_frame_addr(struct tegra_ivc *ivc,
88 struct tegra_ivc_channel_header *h,
91 BUG_ON(frame >= ivc->nframes);
93 return ((ulong)h) + sizeof(struct tegra_ivc_channel_header) +
94 (ivc->frame_size * frame);
97 static inline void *tegra_ivc_frame_pointer(struct tegra_ivc *ivc,
98 struct tegra_ivc_channel_header *ch,
101 return (void *)tegra_ivc_frame_addr(ivc, ch, frame);
104 static inline void tegra_ivc_invalidate_frame(struct tegra_ivc *ivc,
105 struct tegra_ivc_channel_header *h,
108 ulong base = tegra_ivc_frame_addr(ivc, h, frame);
109 invalidate_dcache_range(base, base + ivc->frame_size);
112 static inline void tegra_ivc_flush_frame(struct tegra_ivc *ivc,
113 struct tegra_ivc_channel_header *h,
116 ulong base = tegra_ivc_frame_addr(ivc, h, frame);
117 flush_dcache_range(base, base + ivc->frame_size);
120 static inline int tegra_ivc_channel_empty(struct tegra_ivc *ivc,
121 struct tegra_ivc_channel_header *ch)
124 * This function performs multiple checks on the same values with
125 * security implications, so create snapshots with ACCESS_ONCE() to
126 * ensure that these checks use the same values.
128 uint32_t w_count = ACCESS_ONCE(ch->w_count);
129 uint32_t r_count = ACCESS_ONCE(ch->r_count);
132 * Perform an over-full check to prevent denial of service attacks where
133 * a server could be easily fooled into believing that there's an
134 * extremely large number of frames ready, since receivers are not
135 * expected to check for full or over-full conditions.
137 * Although the channel isn't empty, this is an invalid case caused by
138 * a potentially malicious peer, so returning empty is safer, because it
139 * gives the impression that the channel has gone silent.
141 if (w_count - r_count > ivc->nframes)
144 return w_count == r_count;
147 static inline int tegra_ivc_channel_full(struct tegra_ivc *ivc,
148 struct tegra_ivc_channel_header *ch)
151 * Invalid cases where the counters indicate that the queue is over
152 * capacity also appear full.
154 return (ACCESS_ONCE(ch->w_count) - ACCESS_ONCE(ch->r_count)) >=
158 static inline void tegra_ivc_advance_rx(struct tegra_ivc *ivc)
160 ACCESS_ONCE(ivc->rx_channel->r_count) =
161 ACCESS_ONCE(ivc->rx_channel->r_count) + 1;
163 if (ivc->r_pos == ivc->nframes - 1)
169 static inline void tegra_ivc_advance_tx(struct tegra_ivc *ivc)
171 ACCESS_ONCE(ivc->tx_channel->w_count) =
172 ACCESS_ONCE(ivc->tx_channel->w_count) + 1;
174 if (ivc->w_pos == ivc->nframes - 1)
180 static inline int tegra_ivc_check_read(struct tegra_ivc *ivc)
185 * tx_channel->state is set locally, so it is not synchronized with
186 * state from the remote peer. The remote peer cannot reset its
187 * transmit counters until we've acknowledged its synchronization
188 * request, so no additional synchronization is required because an
189 * asynchronous transition of rx_channel->state to ivc_state_ack is not
192 if (ivc->tx_channel->state != ivc_state_established)
196 * Avoid unnecessary invalidations when performing repeated accesses to
197 * an IVC channel by checking the old queue pointers first.
198 * Synchronization is only necessary when these pointers indicate empty
201 if (!tegra_ivc_channel_empty(ivc, ivc->rx_channel))
204 offset = offsetof(struct tegra_ivc_channel_header, w_count);
205 tegra_ivc_invalidate_counter(ivc, ivc->rx_channel, offset);
206 return tegra_ivc_channel_empty(ivc, ivc->rx_channel) ? -ENOMEM : 0;
209 static inline int tegra_ivc_check_write(struct tegra_ivc *ivc)
213 if (ivc->tx_channel->state != ivc_state_established)
216 if (!tegra_ivc_channel_full(ivc, ivc->tx_channel))
219 offset = offsetof(struct tegra_ivc_channel_header, r_count);
220 tegra_ivc_invalidate_counter(ivc, ivc->tx_channel, offset);
221 return tegra_ivc_channel_full(ivc, ivc->tx_channel) ? -ENOMEM : 0;
224 static inline uint32_t tegra_ivc_channel_avail_count(struct tegra_ivc *ivc,
225 struct tegra_ivc_channel_header *ch)
228 * This function isn't expected to be used in scenarios where an
229 * over-full situation can lead to denial of service attacks. See the
230 * comment in tegra_ivc_channel_empty() for an explanation about
231 * special over-full considerations.
233 return ACCESS_ONCE(ch->w_count) - ACCESS_ONCE(ch->r_count);
236 int tegra_ivc_read_get_next_frame(struct tegra_ivc *ivc, void **frame)
238 int result = tegra_ivc_check_read(ivc);
243 * Order observation of w_pos potentially indicating new data before
248 tegra_ivc_invalidate_frame(ivc, ivc->rx_channel, ivc->r_pos);
249 *frame = tegra_ivc_frame_pointer(ivc, ivc->rx_channel, ivc->r_pos);
254 int tegra_ivc_read_advance(struct tegra_ivc *ivc)
260 * No read barriers or synchronization here: the caller is expected to
261 * have already observed the channel non-empty. This check is just to
262 * catch programming errors.
264 result = tegra_ivc_check_read(ivc);
268 tegra_ivc_advance_rx(ivc);
269 offset = offsetof(struct tegra_ivc_channel_header, r_count);
270 tegra_ivc_flush_counter(ivc, ivc->rx_channel, offset);
273 * Ensure our write to r_pos occurs before our read from w_pos.
277 offset = offsetof(struct tegra_ivc_channel_header, w_count);
278 tegra_ivc_invalidate_counter(ivc, ivc->rx_channel, offset);
280 if (tegra_ivc_channel_avail_count(ivc, ivc->rx_channel) ==
287 int tegra_ivc_write_get_next_frame(struct tegra_ivc *ivc, void **frame)
289 int result = tegra_ivc_check_write(ivc);
293 *frame = tegra_ivc_frame_pointer(ivc, ivc->tx_channel, ivc->w_pos);
298 int tegra_ivc_write_advance(struct tegra_ivc *ivc)
303 result = tegra_ivc_check_write(ivc);
307 tegra_ivc_flush_frame(ivc, ivc->tx_channel, ivc->w_pos);
310 * Order any possible stores to the frame before update of w_pos.
314 tegra_ivc_advance_tx(ivc);
315 offset = offsetof(struct tegra_ivc_channel_header, w_count);
316 tegra_ivc_flush_counter(ivc, ivc->tx_channel, offset);
319 * Ensure our write to w_pos occurs before our read from r_pos.
323 offset = offsetof(struct tegra_ivc_channel_header, r_count);
324 tegra_ivc_invalidate_counter(ivc, ivc->tx_channel, offset);
326 if (tegra_ivc_channel_avail_count(ivc, ivc->tx_channel) == 1)
333 * ===============================================================
334 * IVC State Transition Table - see tegra_ivc_channel_notified()
335 * ===============================================================
337 * local remote action
338 * ----- ------ -----------------------------------
340 * SYNC ACK reset counters; move to EST; notify
341 * SYNC SYNC reset counters; move to ACK; notify
342 * ACK EST move to EST; notify
343 * ACK ACK move to EST; notify
344 * ACK SYNC reset counters; move to ACK; notify
347 * EST SYNC reset counters; move to ACK; notify
349 * ===============================================================
351 int tegra_ivc_channel_notified(struct tegra_ivc *ivc)
354 enum ivc_state peer_state;
356 /* Copy the receiver's state out of shared memory. */
357 offset = offsetof(struct tegra_ivc_channel_header, w_count);
358 tegra_ivc_invalidate_counter(ivc, ivc->rx_channel, offset);
359 peer_state = ACCESS_ONCE(ivc->rx_channel->state);
361 if (peer_state == ivc_state_sync) {
363 * Order observation of ivc_state_sync before stores clearing
369 * Reset tx_channel counters. The remote end is in the SYNC
370 * state and won't make progress until we change our state,
371 * so the counters are not in use at this time.
373 ivc->tx_channel->w_count = 0;
374 ivc->rx_channel->r_count = 0;
380 * Ensure that counters appear cleared before new state can be
386 * Move to ACK state. We have just cleared our counters, so it
387 * is now safe for the remote end to start using these values.
389 ivc->tx_channel->state = ivc_state_ack;
390 offset = offsetof(struct tegra_ivc_channel_header, w_count);
391 tegra_ivc_flush_counter(ivc, ivc->tx_channel, offset);
394 * Notify remote end to observe state transition.
397 } else if (ivc->tx_channel->state == ivc_state_sync &&
398 peer_state == ivc_state_ack) {
400 * Order observation of ivc_state_sync before stores clearing
406 * Reset tx_channel counters. The remote end is in the ACK
407 * state and won't make progress until we change our state,
408 * so the counters are not in use at this time.
410 ivc->tx_channel->w_count = 0;
411 ivc->rx_channel->r_count = 0;
417 * Ensure that counters appear cleared before new state can be
423 * Move to ESTABLISHED state. We know that the remote end has
424 * already cleared its counters, so it is safe to start
425 * writing/reading on this channel.
427 ivc->tx_channel->state = ivc_state_established;
428 offset = offsetof(struct tegra_ivc_channel_header, w_count);
429 tegra_ivc_flush_counter(ivc, ivc->tx_channel, offset);
432 * Notify remote end to observe state transition.
435 } else if (ivc->tx_channel->state == ivc_state_ack) {
437 * At this point, we have observed the peer to be in either
438 * the ACK or ESTABLISHED state. Next, order observation of
439 * peer state before storing to tx_channel.
444 * Move to ESTABLISHED state. We know that we have previously
445 * cleared our counters, and we know that the remote end has
446 * cleared its counters, so it is safe to start writing/reading
449 ivc->tx_channel->state = ivc_state_established;
450 offset = offsetof(struct tegra_ivc_channel_header, w_count);
451 tegra_ivc_flush_counter(ivc, ivc->tx_channel, offset);
454 * Notify remote end to observe state transition.
459 * There is no need to handle any further action. Either the
460 * channel is already fully established, or we are waiting for
461 * the remote end to catch up with our current state. Refer
462 * to the diagram in "IVC State Transition Table" above.
466 if (ivc->tx_channel->state != ivc_state_established)
472 void tegra_ivc_channel_reset(struct tegra_ivc *ivc)
476 ivc->tx_channel->state = ivc_state_sync;
477 offset = offsetof(struct tegra_ivc_channel_header, w_count);
478 tegra_ivc_flush_counter(ivc, ivc->tx_channel, offset);
482 static int check_ivc_params(ulong qbase1, ulong qbase2, uint32_t nframes,
487 BUG_ON(offsetof(struct tegra_ivc_channel_header, w_count) &
488 (TEGRA_IVC_ALIGN - 1));
489 BUG_ON(offsetof(struct tegra_ivc_channel_header, r_count) &
490 (TEGRA_IVC_ALIGN - 1));
491 BUG_ON(sizeof(struct tegra_ivc_channel_header) &
492 (TEGRA_IVC_ALIGN - 1));
494 if ((uint64_t)nframes * (uint64_t)frame_size >= 0x100000000) {
495 pr_err("tegra_ivc: nframes * frame_size overflows\n");
500 * The headers must at least be aligned enough for counters
501 * to be accessed atomically.
503 if ((qbase1 & (TEGRA_IVC_ALIGN - 1)) ||
504 (qbase2 & (TEGRA_IVC_ALIGN - 1))) {
505 pr_err("tegra_ivc: channel start not aligned\n");
509 if (frame_size & (TEGRA_IVC_ALIGN - 1)) {
510 pr_err("tegra_ivc: frame size not adequately aligned\n");
514 if (qbase1 < qbase2) {
515 if (qbase1 + frame_size * nframes > qbase2)
518 if (qbase2 + frame_size * nframes > qbase1)
523 pr_err("tegra_ivc: queue regions overlap\n");
530 int tegra_ivc_init(struct tegra_ivc *ivc, ulong rx_base, ulong tx_base,
531 uint32_t nframes, uint32_t frame_size,
532 void (*notify)(struct tegra_ivc *))
539 ret = check_ivc_params(rx_base, tx_base, nframes, frame_size);
543 ivc->rx_channel = (struct tegra_ivc_channel_header *)rx_base;
544 ivc->tx_channel = (struct tegra_ivc_channel_header *)tx_base;
547 ivc->nframes = nframes;
548 ivc->frame_size = frame_size;
549 ivc->notify = notify;