1 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
2 /* Copyright (c) 2010-2012 Broadcom. All rights reserved. */
7 #include <linux/mutex.h>
8 #include <linux/completion.h>
9 #include <linux/kthread.h>
10 #include <linux/wait.h>
12 #include "vchiq_cfg.h"
16 /* Run time control of log level, based on KERN_XXX level. */
17 #define VCHIQ_LOG_DEFAULT 4
18 #define VCHIQ_LOG_ERROR 3
19 #define VCHIQ_LOG_WARNING 4
20 #define VCHIQ_LOG_INFO 6
21 #define VCHIQ_LOG_TRACE 7
23 #define VCHIQ_LOG_PREFIX KERN_INFO "vchiq: "
25 #ifndef vchiq_log_error
26 #define vchiq_log_error(cat, fmt, ...) \
27 do { if (cat >= VCHIQ_LOG_ERROR) \
28 printk(VCHIQ_LOG_PREFIX fmt "\n", ##__VA_ARGS__); } while (0)
30 #ifndef vchiq_log_warning
31 #define vchiq_log_warning(cat, fmt, ...) \
32 do { if (cat >= VCHIQ_LOG_WARNING) \
33 printk(VCHIQ_LOG_PREFIX fmt "\n", ##__VA_ARGS__); } while (0)
35 #ifndef vchiq_log_info
36 #define vchiq_log_info(cat, fmt, ...) \
37 do { if (cat >= VCHIQ_LOG_INFO) \
38 printk(VCHIQ_LOG_PREFIX fmt "\n", ##__VA_ARGS__); } while (0)
40 #ifndef vchiq_log_trace
41 #define vchiq_log_trace(cat, fmt, ...) \
42 do { if (cat >= VCHIQ_LOG_TRACE) \
43 printk(VCHIQ_LOG_PREFIX fmt "\n", ##__VA_ARGS__); } while (0)
46 #define vchiq_loud_error(...) \
47 vchiq_log_error(vchiq_core_log_level, "===== " __VA_ARGS__)
49 #ifndef vchiq_static_assert
50 #define vchiq_static_assert(cond) __attribute__((unused)) \
51 extern int vchiq_static_assert[(cond) ? 1 : -1]
54 #define IS_POW2(x) (x && ((x & (x - 1)) == 0))
56 /* Ensure that the slot size and maximum number of slots are powers of 2 */
57 vchiq_static_assert(IS_POW2(VCHIQ_SLOT_SIZE));
58 vchiq_static_assert(IS_POW2(VCHIQ_MAX_SLOTS));
59 vchiq_static_assert(IS_POW2(VCHIQ_MAX_SLOTS_PER_SIDE));
61 #define VCHIQ_SLOT_MASK (VCHIQ_SLOT_SIZE - 1)
62 #define VCHIQ_SLOT_QUEUE_MASK (VCHIQ_MAX_SLOTS_PER_SIDE - 1)
63 #define VCHIQ_SLOT_ZERO_SLOTS ((sizeof(struct vchiq_slot_zero) + \
64 VCHIQ_SLOT_SIZE - 1) / VCHIQ_SLOT_SIZE)
66 #define VCHIQ_MSG_PADDING 0 /* - */
67 #define VCHIQ_MSG_CONNECT 1 /* - */
68 #define VCHIQ_MSG_OPEN 2 /* + (srcport, -), fourcc, client_id */
69 #define VCHIQ_MSG_OPENACK 3 /* + (srcport, dstport) */
70 #define VCHIQ_MSG_CLOSE 4 /* + (srcport, dstport) */
71 #define VCHIQ_MSG_DATA 5 /* + (srcport, dstport) */
72 #define VCHIQ_MSG_BULK_RX 6 /* + (srcport, dstport), data, size */
73 #define VCHIQ_MSG_BULK_TX 7 /* + (srcport, dstport), data, size */
74 #define VCHIQ_MSG_BULK_RX_DONE 8 /* + (srcport, dstport), actual */
75 #define VCHIQ_MSG_BULK_TX_DONE 9 /* + (srcport, dstport), actual */
76 #define VCHIQ_MSG_PAUSE 10 /* - */
77 #define VCHIQ_MSG_RESUME 11 /* - */
78 #define VCHIQ_MSG_REMOTE_USE 12 /* - */
79 #define VCHIQ_MSG_REMOTE_RELEASE 13 /* - */
80 #define VCHIQ_MSG_REMOTE_USE_ACTIVE 14 /* - */
82 #define VCHIQ_PORT_MAX (VCHIQ_MAX_SERVICES - 1)
83 #define VCHIQ_PORT_FREE 0x1000
84 #define VCHIQ_PORT_IS_VALID(port) (port < VCHIQ_PORT_FREE)
85 #define VCHIQ_MAKE_MSG(type, srcport, dstport) \
86 ((type<<24) | (srcport<<12) | (dstport<<0))
87 #define VCHIQ_MSG_TYPE(msgid) ((unsigned int)msgid >> 24)
88 #define VCHIQ_MSG_SRCPORT(msgid) \
89 (unsigned short)(((unsigned int)msgid >> 12) & 0xfff)
90 #define VCHIQ_MSG_DSTPORT(msgid) \
91 ((unsigned short)msgid & 0xfff)
93 #define VCHIQ_FOURCC_AS_4CHARS(fourcc) \
94 ((fourcc) >> 24) & 0xff, \
95 ((fourcc) >> 16) & 0xff, \
96 ((fourcc) >> 8) & 0xff, \
99 /* Ensure the fields are wide enough */
100 vchiq_static_assert(VCHIQ_MSG_SRCPORT(VCHIQ_MAKE_MSG(0, 0, VCHIQ_PORT_MAX))
102 vchiq_static_assert(VCHIQ_MSG_TYPE(VCHIQ_MAKE_MSG(0, VCHIQ_PORT_MAX, 0)) == 0);
103 vchiq_static_assert((unsigned int)VCHIQ_PORT_MAX <
104 (unsigned int)VCHIQ_PORT_FREE);
106 #define VCHIQ_MSGID_PADDING VCHIQ_MAKE_MSG(VCHIQ_MSG_PADDING, 0, 0)
107 #define VCHIQ_MSGID_CLAIMED 0x40000000
109 #define VCHIQ_FOURCC_INVALID 0x00000000
110 #define VCHIQ_FOURCC_IS_LEGAL(fourcc) (fourcc != VCHIQ_FOURCC_INVALID)
112 #define VCHIQ_BULK_ACTUAL_ABORTED -1
114 typedef uint32_t BITSET_T;
116 vchiq_static_assert((sizeof(BITSET_T) * 8) == 32);
118 #define BITSET_SIZE(b) ((b + 31) >> 5)
119 #define BITSET_WORD(b) (b >> 5)
120 #define BITSET_BIT(b) (1 << (b & 31))
121 #define BITSET_IS_SET(bs, b) (bs[BITSET_WORD(b)] & BITSET_BIT(b))
122 #define BITSET_SET(bs, b) (bs[BITSET_WORD(b)] |= BITSET_BIT(b))
123 #define BITSET_CLR(bs, b) (bs[BITSET_WORD(b)] &= ~BITSET_BIT(b))
125 #if VCHIQ_ENABLE_STATS
126 #define VCHIQ_STATS_INC(state, stat) (state->stats. stat++)
127 #define VCHIQ_SERVICE_STATS_INC(service, stat) (service->stats. stat++)
128 #define VCHIQ_SERVICE_STATS_ADD(service, stat, addend) \
129 (service->stats. stat += addend)
131 #define VCHIQ_STATS_INC(state, stat) ((void)0)
132 #define VCHIQ_SERVICE_STATS_INC(service, stat) ((void)0)
133 #define VCHIQ_SERVICE_STATS_ADD(service, stat, addend) ((void)0)
138 #if VCHIQ_ENABLE_DEBUG
139 DEBUG_SLOT_HANDLER_COUNT,
140 DEBUG_SLOT_HANDLER_LINE,
144 DEBUG_AWAIT_COMPLETION_LINE,
145 DEBUG_DEQUEUE_MESSAGE_LINE,
146 DEBUG_SERVICE_CALLBACK_LINE,
147 DEBUG_MSG_QUEUE_FULL_COUNT,
148 DEBUG_COMPLETION_QUEUE_FULL_COUNT,
153 #if VCHIQ_ENABLE_DEBUG
155 #define DEBUG_INITIALISE(local) int *debug_ptr = (local)->debug;
156 #define DEBUG_TRACE(d) \
157 do { debug_ptr[DEBUG_ ## d] = __LINE__; dsb(sy); } while (0)
158 #define DEBUG_VALUE(d, v) \
159 do { debug_ptr[DEBUG_ ## d] = (v); dsb(sy); } while (0)
160 #define DEBUG_COUNT(d) \
161 do { debug_ptr[DEBUG_ ## d]++; dsb(sy); } while (0)
163 #else /* VCHIQ_ENABLE_DEBUG */
165 #define DEBUG_INITIALISE(local)
166 #define DEBUG_TRACE(d)
167 #define DEBUG_VALUE(d, v)
168 #define DEBUG_COUNT(d)
170 #endif /* VCHIQ_ENABLE_DEBUG */
173 VCHIQ_CONNSTATE_DISCONNECTED,
174 VCHIQ_CONNSTATE_CONNECTING,
175 VCHIQ_CONNSTATE_CONNECTED,
176 VCHIQ_CONNSTATE_PAUSING,
177 VCHIQ_CONNSTATE_PAUSE_SENT,
178 VCHIQ_CONNSTATE_PAUSED,
179 VCHIQ_CONNSTATE_RESUMING,
180 VCHIQ_CONNSTATE_PAUSE_TIMEOUT,
181 VCHIQ_CONNSTATE_RESUME_TIMEOUT
186 VCHIQ_SRVSTATE_HIDDEN,
187 VCHIQ_SRVSTATE_LISTENING,
188 VCHIQ_SRVSTATE_OPENING,
190 VCHIQ_SRVSTATE_OPENSYNC,
191 VCHIQ_SRVSTATE_CLOSESENT,
192 VCHIQ_SRVSTATE_CLOSERECVD,
193 VCHIQ_SRVSTATE_CLOSEWAIT,
194 VCHIQ_SRVSTATE_CLOSED
198 VCHIQ_POLL_TERMINATE,
210 typedef void (*VCHIQ_USERDATA_TERM_T)(void *userdata);
223 struct vchiq_bulk_queue {
224 int local_insert; /* Where to insert the next local bulk */
225 int remote_insert; /* Where to insert the next remote bulk (master) */
226 int process; /* Bulk to transfer next */
227 int remote_notify; /* Bulk to notify the remote client of next (mstr) */
228 int remove; /* Bulk to notify the local client of, and remove,
230 struct vchiq_bulk bulks[VCHIQ_NUM_SERVICE_BULKS];
233 struct remote_event {
239 typedef struct opaque_platform_state_t *VCHIQ_PLATFORM_STATE_T;
242 char data[VCHIQ_SLOT_SIZE];
245 struct vchiq_slot_info {
246 /* Use two counters rather than one to avoid the need for a mutex. */
251 struct vchiq_service {
252 struct vchiq_service_base base;
253 VCHIQ_SERVICE_HANDLE_T handle;
254 unsigned int ref_count;
256 VCHIQ_USERDATA_TERM_T userdata_term;
257 unsigned int localport;
258 unsigned int remoteport;
270 struct vchiq_state *state;
271 VCHIQ_INSTANCE_T instance;
273 int service_use_count;
275 struct vchiq_bulk_queue bulk_tx;
276 struct vchiq_bulk_queue bulk_rx;
278 struct completion remove_event;
279 struct completion bulk_remove_event;
280 struct mutex bulk_mutex;
282 struct service_stats_struct {
291 int bulk_aborted_count;
292 uint64_t ctrl_tx_bytes;
293 uint64_t ctrl_rx_bytes;
294 uint64_t bulk_tx_bytes;
295 uint64_t bulk_rx_bytes;
299 /* The quota information is outside struct vchiq_service so that it can
300 * be statically allocated, since for accounting reasons a service's slot
301 * usage is carried over between users of the same port number.
303 struct vchiq_service_quota {
304 unsigned short slot_quota;
305 unsigned short slot_use_count;
306 unsigned short message_quota;
307 unsigned short message_use_count;
308 struct completion quota_event;
309 int previous_tx_index;
312 struct vchiq_shared_state {
314 /* A non-zero value here indicates that the content is valid. */
317 /* The first and last (inclusive) slots allocated to the owner. */
321 /* The slot allocated to synchronous messages from the owner. */
324 /* Signalling this event indicates that owner's slot handler thread
326 struct remote_event trigger;
328 /* Indicates the byte position within the stream where the next message
329 ** will be written. The least significant bits are an index into the
330 ** slot. The next bits are the index of the slot in slot_queue. */
333 /* This event should be signalled when a slot is recycled. */
334 struct remote_event recycle;
336 /* The slot_queue index where the next recycled slot will be written. */
337 int slot_queue_recycle;
339 /* This event should be signalled when a synchronous message is sent. */
340 struct remote_event sync_trigger;
342 /* This event should be signalled when a synchronous message has been
344 struct remote_event sync_release;
346 /* A circular buffer of slot indexes. */
347 int slot_queue[VCHIQ_MAX_SLOTS_PER_SIDE];
349 /* Debugging state */
350 int debug[DEBUG_MAX];
353 struct vchiq_slot_zero {
360 int max_slots_per_side;
361 int platform_data[2];
362 struct vchiq_shared_state master;
363 struct vchiq_shared_state slave;
364 struct vchiq_slot_info slots[VCHIQ_MAX_SLOTS];
370 VCHIQ_CONNSTATE_T conn_state;
371 short version_common;
373 struct vchiq_shared_state *local;
374 struct vchiq_shared_state *remote;
375 struct vchiq_slot *slot_data;
377 unsigned short default_slot_quota;
378 unsigned short default_message_quota;
380 /* Event indicating connect message received */
381 struct completion connect;
383 /* Mutex protecting services */
385 VCHIQ_INSTANCE_T *instance;
387 /* Processes incoming messages */
388 struct task_struct *slot_handler_thread;
390 /* Processes recycled slots */
391 struct task_struct *recycle_thread;
393 /* Processes synchronous messages */
394 struct task_struct *sync_thread;
396 /* Local implementation of the trigger remote event */
397 wait_queue_head_t trigger_event;
399 /* Local implementation of the recycle remote event */
400 wait_queue_head_t recycle_event;
402 /* Local implementation of the sync trigger remote event */
403 wait_queue_head_t sync_trigger_event;
405 /* Local implementation of the sync release remote event */
406 wait_queue_head_t sync_release_event;
410 struct vchiq_slot_info *rx_info;
412 struct mutex slot_mutex;
414 struct mutex recycle_mutex;
416 struct mutex sync_mutex;
418 struct mutex bulk_transfer_mutex;
420 /* Indicates the byte position within the stream from where the next
421 ** message will be read. The least significant bits are an index into
422 ** the slot.The next bits are the index of the slot in
423 ** remote->slot_queue. */
426 /* A cached copy of local->tx_pos. Only write to local->tx_pos, and read
427 from remote->tx_pos. */
430 /* The slot_queue index of the slot to become available next. */
431 int slot_queue_available;
433 /* A flag to indicate if any poll has been requested */
436 /* Ths index of the previous slot used for data messages. */
437 int previous_data_index;
439 /* The number of slots occupied by data messages. */
440 unsigned short data_use_count;
442 /* The maximum number of slots to be occupied by data messages. */
443 unsigned short data_quota;
445 /* An array of bit sets indicating which services must be polled. */
446 atomic_t poll_services[BITSET_SIZE(VCHIQ_MAX_SERVICES)];
448 /* The number of the first unused service */
451 /* Signalled when a free slot becomes available. */
452 struct completion slot_available_event;
454 struct completion slot_remove_event;
456 /* Signalled when a free data slot becomes available. */
457 struct completion data_quota_event;
459 struct state_stats_struct {
467 struct vchiq_service *services[VCHIQ_MAX_SERVICES];
468 struct vchiq_service_quota service_quotas[VCHIQ_MAX_SERVICES];
469 struct vchiq_slot_info slot_info[VCHIQ_MAX_SLOTS];
471 VCHIQ_PLATFORM_STATE_T platform_state;
475 struct vchiq_bulk *bulk;
476 struct completion event;
480 extern spinlock_t bulk_waiter_spinlock;
482 extern int vchiq_core_log_level;
483 extern int vchiq_core_msg_log_level;
484 extern int vchiq_sync_log_level;
486 extern struct vchiq_state *vchiq_states[VCHIQ_MAX_STATES];
489 get_conn_state_name(VCHIQ_CONNSTATE_T conn_state);
491 extern struct vchiq_slot_zero *
492 vchiq_init_slots(void *mem_base, int mem_size);
494 extern VCHIQ_STATUS_T
495 vchiq_init_state(struct vchiq_state *state, struct vchiq_slot_zero *slot_zero);
497 extern VCHIQ_STATUS_T
498 vchiq_connect_internal(struct vchiq_state *state, VCHIQ_INSTANCE_T instance);
500 extern struct vchiq_service *
501 vchiq_add_service_internal(struct vchiq_state *state,
502 const struct vchiq_service_params *params,
503 int srvstate, VCHIQ_INSTANCE_T instance,
504 VCHIQ_USERDATA_TERM_T userdata_term);
506 extern VCHIQ_STATUS_T
507 vchiq_open_service_internal(struct vchiq_service *service, int client_id);
509 extern VCHIQ_STATUS_T
510 vchiq_close_service_internal(struct vchiq_service *service, int close_recvd);
513 vchiq_terminate_service_internal(struct vchiq_service *service);
516 vchiq_free_service_internal(struct vchiq_service *service);
518 extern VCHIQ_STATUS_T
519 vchiq_shutdown_internal(struct vchiq_state *state, VCHIQ_INSTANCE_T instance);
522 remote_event_pollall(struct vchiq_state *state);
524 extern VCHIQ_STATUS_T
525 vchiq_bulk_transfer(VCHIQ_SERVICE_HANDLE_T handle, void *offset, int size,
526 void *userdata, VCHIQ_BULK_MODE_T mode,
527 VCHIQ_BULK_DIR_T dir);
530 vchiq_dump_state(void *dump_context, struct vchiq_state *state);
533 vchiq_dump_service_state(void *dump_context, struct vchiq_service *service);
536 vchiq_loud_error_header(void);
539 vchiq_loud_error_footer(void);
542 request_poll(struct vchiq_state *state, struct vchiq_service *service,
545 static inline struct vchiq_service *
546 handle_to_service(VCHIQ_SERVICE_HANDLE_T handle)
548 struct vchiq_state *state = vchiq_states[(handle / VCHIQ_MAX_SERVICES) &
549 (VCHIQ_MAX_STATES - 1)];
553 return state->services[handle & (VCHIQ_MAX_SERVICES - 1)];
556 extern struct vchiq_service *
557 find_service_by_handle(VCHIQ_SERVICE_HANDLE_T handle);
559 extern struct vchiq_service *
560 find_service_by_port(struct vchiq_state *state, int localport);
562 extern struct vchiq_service *
563 find_service_for_instance(VCHIQ_INSTANCE_T instance,
564 VCHIQ_SERVICE_HANDLE_T handle);
566 extern struct vchiq_service *
567 find_closed_service_for_instance(VCHIQ_INSTANCE_T instance,
568 VCHIQ_SERVICE_HANDLE_T handle);
570 extern struct vchiq_service *
571 next_service_by_instance(struct vchiq_state *state, VCHIQ_INSTANCE_T instance,
575 lock_service(struct vchiq_service *service);
578 unlock_service(struct vchiq_service *service);
580 /* The following functions are called from vchiq_core, and external
581 ** implementations must be provided. */
583 extern VCHIQ_STATUS_T
584 vchiq_prepare_bulk_data(struct vchiq_bulk *bulk, void *offset, int size,
588 vchiq_complete_bulk(struct vchiq_bulk *bulk);
591 remote_event_signal(struct remote_event *event);
594 vchiq_platform_check_suspend(struct vchiq_state *state);
597 vchiq_platform_paused(struct vchiq_state *state);
599 extern VCHIQ_STATUS_T
600 vchiq_platform_resume(struct vchiq_state *state);
603 vchiq_platform_resumed(struct vchiq_state *state);
606 vchiq_dump(void *dump_context, const char *str, int len);
609 vchiq_dump_platform_state(void *dump_context);
612 vchiq_dump_platform_instances(void *dump_context);
615 vchiq_dump_platform_service_state(void *dump_context,
616 struct vchiq_service *service);
618 extern VCHIQ_STATUS_T
619 vchiq_use_service_internal(struct vchiq_service *service);
621 extern VCHIQ_STATUS_T
622 vchiq_release_service_internal(struct vchiq_service *service);
625 vchiq_on_remote_use(struct vchiq_state *state);
628 vchiq_on_remote_release(struct vchiq_state *state);
630 extern VCHIQ_STATUS_T
631 vchiq_platform_init_state(struct vchiq_state *state);
633 extern VCHIQ_STATUS_T
634 vchiq_check_service(struct vchiq_service *service);
637 vchiq_on_remote_use_active(struct vchiq_state *state);
639 extern VCHIQ_STATUS_T
640 vchiq_send_remote_use(struct vchiq_state *state);
642 extern VCHIQ_STATUS_T
643 vchiq_send_remote_use_active(struct vchiq_state *state);
646 vchiq_platform_conn_state_changed(struct vchiq_state *state,
647 VCHIQ_CONNSTATE_T oldstate,
648 VCHIQ_CONNSTATE_T newstate);
651 vchiq_platform_handle_timeout(struct vchiq_state *state);
654 vchiq_set_conn_state(struct vchiq_state *state, VCHIQ_CONNSTATE_T newstate);
657 vchiq_log_dump_mem(const char *label, uint32_t addr, const void *voidMem,