drivers/usb/cdns3/gadget.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Cadence USBSS DRD Driver - gadget side.
   4  *
   5  * Copyright (C) 2018-2019 Cadence Design Systems.
   6  * Copyright (C) 2017-2018 NXP
   7  *
   8  * Authors: Pawel Jez <pjez@cadence.com>,
   9  *          Pawel Laszczak <pawell@cadence.com>
  10  *          Peter Chen <peter.chen@nxp.com>
  11  */
  12
  13 /*
  14  * Work around 1:
  15  * At some situations, the controller may get stale data address in TRB
  16  * at below sequences:
  17  * 1. Controller read TRB includes data address
  18  * 2. Software updates TRBs includes data address and Cycle bit
  19  * 3. Controller read TRB which includes Cycle bit
  20  * 4. DMA run with stale data address
  21  *
  22  * To fix this problem, driver needs to make the first TRB in TD as invalid.
  23  * After preparing all TRBs driver needs to check the position of DMA and
  24  * if the DMA point to the first just added TRB and doorbell is 1,
  25  * then driver must defer making this TRB as valid. This TRB will be make
  26  * as valid during adding next TRB only if DMA is stopped or at TRBERR
  27  * interrupt.
  28  *
  29  * Issue has been fixed in DEV_VER_V3 version of controller.
  30  *
  31  * Work around 2:
  32  * Controller for OUT endpoints has shared on-chip buffers for all incoming
  33  * packets, including ep0out. It's FIFO buffer, so packets must be handle by DMA
  34  * in correct order. If the first packet in the buffer will not be handled,
  35  * then the following packets directed for other endpoints and  functions
  36  * will be blocked.
  37  * Additionally the packets directed to one endpoint can block entire on-chip
  38  * buffers. In this case transfer to other endpoints also will blocked.
  39  *
  40  * To resolve this issue after raising the descriptor missing interrupt
  41  * driver prepares internal usb_request object and use it to arm DMA transfer.
  42  *
  43  * The problematic situation was observed in case when endpoint has been enabled
  44  * but no usb_request were queued. Driver try detects such endpoints and will
  45  * use this workaround only for these endpoint.
  46  *
  47  * Driver use limited number of buffer. This number can be set by macro
  48  * CDNS3_WA2_NUM_BUFFERS.
  49  *
  50  * Such blocking situation was observed on ACM gadget. For this function
  51  * host send OUT data packet but ACM function is not prepared for this packet.
  52  * It's cause that buffer placed in on chip memory block transfer to other
  53  * endpoints.
  54  *
  55  * Issue has been fixed in DEV_VER_V2 version of controller.
  56  *
  57  */
  58
  59 #include <dm.h>
  60 #include <dm/device_compat.h>
  61 #include <dm/devres.h>
  62 #include <linux/delay.h>
  63 #include <linux/err.h>
  64 #include <linux/usb/gadget.h>
  65 #include <linux/compat.h>
  66 #include <linux/iopoll.h>
  67 #include <linux/dma-mapping.h>
  68 #include <linux/bitmap.h>
  69 #include <linux/bug.h>
  70
  71 #include "core.h"
  72 #include "gadget-export.h"
  73 #include "gadget.h"
  74 #include "trace.h"
  75 #include "drd.h"
  76
  77 #define readl_poll_timeout_atomic readl_poll_timeout
  78 #define usleep_range(a, b) udelay((b))
  79
  80 static int __cdns3_gadget_ep_queue(struct usb_ep *ep,
  81                                    struct usb_request *request,
  82                                    gfp_t gfp_flags);
  83
  84 /**
  85  * cdns3_set_register_bit - set bit in given register.
  86  * @ptr: address of device controller register to be read and changed
  87  * @mask: bits requested to set
  88  */
  89 void cdns3_set_register_bit(void __iomem *ptr, u32 mask)
  90 {
  91         mask = readl(ptr) | mask;
  92         writel(mask, ptr);
  93 }
  94
  95 /**
  96  * cdns3_ep_addr_to_index - Macro converts endpoint address to
  97  * index of endpoint object in cdns3_device.eps[] container
  98  * @ep_addr: endpoint address for which endpoint object is required
  99  *
 100  */
 101 u8 cdns3_ep_addr_to_index(u8 ep_addr)
 102 {
 103         return (((ep_addr & 0x7F)) + ((ep_addr & USB_DIR_IN) ? 16 : 0));
 104 }
 105
 106 static int cdns3_get_dma_pos(struct cdns3_device *priv_dev,
 107                              struct cdns3_endpoint *priv_ep)
 108 {
 109         int dma_index;
 110
 111         dma_index = readl(&priv_dev->regs->ep_traddr) - priv_ep->trb_pool_dma;
 112
 113         return dma_index / TRB_SIZE;
 114 }
 115
 116 /**
 117  * cdns3_next_request - returns next request from list
 118  * @list: list containing requests
 119  *
 120  * Returns request or NULL if no requests in list
 121  */
 122 struct usb_request *cdns3_next_request(struct list_head *list)
 123 {
 124         return list_first_entry_or_null(list, struct usb_request, list);
 125 }
 126
 127 /**
 128  * cdns3_next_align_buf - returns next buffer from list
 129  * @list: list containing buffers
 130  *
 131  * Returns buffer or NULL if no buffers in list
 132  */
 133 struct cdns3_aligned_buf *cdns3_next_align_buf(struct list_head *list)
 134 {
 135         return list_first_entry_or_null(list, struct cdns3_aligned_buf, list);
 136 }
 137
 138 /**
 139  * cdns3_next_priv_request - returns next request from list
 140  * @list: list containing requests
 141  *
 142  * Returns request or NULL if no requests in list
 143  */
 144 struct cdns3_request *cdns3_next_priv_request(struct list_head *list)
 145 {
 146         return list_first_entry_or_null(list, struct cdns3_request, list);
 147 }
 148
 149 /**
 150  * select_ep - selects endpoint
 151  * @priv_dev:  extended gadget object
 152  * @ep: endpoint address
 153  */
 154 void cdns3_select_ep(struct cdns3_device *priv_dev, u32 ep)
 155 {
 156         if (priv_dev->selected_ep == ep)
 157                 return;
 158
 159         priv_dev->selected_ep = ep;
 160         writel(ep, &priv_dev->regs->ep_sel);
 161 }
 162
 163 dma_addr_t cdns3_trb_virt_to_dma(struct cdns3_endpoint *priv_ep,
 164                                  struct cdns3_trb *trb)
 165 {
 166         u32 offset = (char *)trb - (char *)priv_ep->trb_pool;
 167
 168         return priv_ep->trb_pool_dma + offset;
 169 }
 170
 171 int cdns3_ring_size(struct cdns3_endpoint *priv_ep)
 172 {
 173         switch (priv_ep->type) {
 174         case USB_ENDPOINT_XFER_ISOC:
 175                 return TRB_ISO_RING_SIZE;
 176         case USB_ENDPOINT_XFER_CONTROL:
 177                 return TRB_CTRL_RING_SIZE;
 178         default:
 179                 return TRB_RING_SIZE;
 180         }
 181 }
 182
 183 /**
 184  * cdns3_allocate_trb_pool - Allocates TRB's pool for selected endpoint
 185  * @priv_ep:  endpoint object
 186  *
 187  * Function will return 0 on success or -ENOMEM on allocation error
 188  */
 189 int cdns3_allocate_trb_pool(struct cdns3_endpoint *priv_ep)
 190 {
 191         int ring_size = cdns3_ring_size(priv_ep);
 192         struct cdns3_trb *link_trb;
 193
 194         if (!priv_ep->trb_pool) {
 195                 priv_ep->trb_pool =
 196                 dma_alloc_coherent(ring_size,
 197                                    (unsigned long *)&priv_ep->trb_pool_dma);
 198                 if (!priv_ep->trb_pool)
 199                         return -ENOMEM;
 200         } else {
 201                 memset(priv_ep->trb_pool, 0, ring_size);
 202         }
 203
 204         if (!priv_ep->num)
 205                 return 0;
 206
 207         priv_ep->num_trbs = ring_size / TRB_SIZE;
 208         /* Initialize the last TRB as Link TRB. */
 209         link_trb = (priv_ep->trb_pool + (priv_ep->num_trbs - 1));
 210         link_trb->buffer = TRB_BUFFER(priv_ep->trb_pool_dma);
 211         link_trb->control = TRB_CYCLE | TRB_TYPE(TRB_LINK) | TRB_TOGGLE;
 212
 213         return 0;
 214 }
 215
 216 static void cdns3_free_trb_pool(struct cdns3_endpoint *priv_ep)
 217 {
 218         if (priv_ep->trb_pool) {
 219                 dma_free_coherent(priv_ep->trb_pool);
 220                 priv_ep->trb_pool = NULL;
 221         }
 222 }
 223
 224 /**
 225  * cdns3_ep_stall_flush - Stalls and flushes selected endpoint
 226  * @priv_ep: endpoint object
 227  *
 228  * Endpoint must be selected before call to this function
 229  */
 230 static void cdns3_ep_stall_flush(struct cdns3_endpoint *priv_ep)
 231 {
 232         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
 233         int val;
 234
 235         trace_cdns3_halt(priv_ep, 1, 1);
 236
 237         writel(EP_CMD_DFLUSH | EP_CMD_ERDY | EP_CMD_SSTALL,
 238                &priv_dev->regs->ep_cmd);
 239
 240         /* wait for DFLUSH cleared */
 241         readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
 242                                   !(val & EP_CMD_DFLUSH), 1000);
 243         priv_ep->flags |= EP_STALLED;
 244         priv_ep->flags &= ~EP_STALL_PENDING;
 245 }
 246
 247 /**
 248  * cdns3_hw_reset_eps_config - reset endpoints configuration kept by controller.
 249  * @priv_dev: extended gadget object
 250  */
 251 void cdns3_hw_reset_eps_config(struct cdns3_device *priv_dev)
 252 {
 253         writel(USB_CONF_CFGRST, &priv_dev->regs->usb_conf);
 254
 255         cdns3_allow_enable_l1(priv_dev, 0);
 256         priv_dev->hw_configured_flag = 0;
 257         priv_dev->onchip_used_size = 0;
 258         priv_dev->out_mem_is_allocated = 0;
 259         priv_dev->wait_for_setup = 0;
 260 }
 261
 262 /**
 263  * cdns3_ep_inc_trb - increment a trb index.
 264  * @index: Pointer to the TRB index to increment.
 265  * @cs: Cycle state
 266  * @trb_in_seg: number of TRBs in segment
 267  *
 268  * The index should never point to the link TRB. After incrementing,
 269  * if it is point to the link TRB, wrap around to the beginning and revert
 270  * cycle state bit The
 271  * link TRB is always at the last TRB entry.
 272  */
 273 static void cdns3_ep_inc_trb(int *index, u8 *cs, int trb_in_seg)
 274 {
 275         (*index)++;
 276         if (*index == (trb_in_seg - 1)) {
 277                 *index = 0;
 278                 *cs ^=  1;
 279         }
 280 }
 281
 282 /**
 283  * cdns3_ep_inc_enq - increment endpoint's enqueue pointer
 284  * @priv_ep: The endpoint whose enqueue pointer we're incrementing
 285  */
 286 static void cdns3_ep_inc_enq(struct cdns3_endpoint *priv_ep)
 287 {
 288         priv_ep->free_trbs--;
 289         cdns3_ep_inc_trb(&priv_ep->enqueue, &priv_ep->pcs, priv_ep->num_trbs);
 290 }
 291
 292 /**
 293  * cdns3_ep_inc_deq - increment endpoint's dequeue pointer
 294  * @priv_ep: The endpoint whose dequeue pointer we're incrementing
 295  */
 296 static void cdns3_ep_inc_deq(struct cdns3_endpoint *priv_ep)
 297 {
 298         priv_ep->free_trbs++;
 299         cdns3_ep_inc_trb(&priv_ep->dequeue, &priv_ep->ccs, priv_ep->num_trbs);
 300 }
 301
 302 void cdns3_move_deq_to_next_trb(struct cdns3_request *priv_req)
 303 {
 304         struct cdns3_endpoint *priv_ep = priv_req->priv_ep;
 305         int current_trb = priv_req->start_trb;
 306
 307         while (current_trb != priv_req->end_trb) {
 308                 cdns3_ep_inc_deq(priv_ep);
 309                 current_trb = priv_ep->dequeue;
 310         }
 311
 312         cdns3_ep_inc_deq(priv_ep);
 313 }
 314
 315 /**
 316  * cdns3_allow_enable_l1 - enable/disable permits to transition to L1.
 317  * @priv_dev: Extended gadget object
 318  * @enable: Enable/disable permit to transition to L1.
 319  *
 320  * If bit USB_CONF_L1EN is set and device receive Extended Token packet,
 321  * then controller answer with ACK handshake.
 322  * If bit USB_CONF_L1DS is set and device receive Extended Token packet,
 323  * then controller answer with NYET handshake.
 324  */
 325 void cdns3_allow_enable_l1(struct cdns3_device *priv_dev, int enable)
 326 {
 327         if (enable)
 328                 writel(USB_CONF_L1EN, &priv_dev->regs->usb_conf);
 329         else
 330                 writel(USB_CONF_L1DS, &priv_dev->regs->usb_conf);
 331 }
 332
 333 enum usb_device_speed cdns3_get_speed(struct cdns3_device *priv_dev)
 334 {
 335         u32 reg;
 336
 337         reg = readl(&priv_dev->regs->usb_sts);
 338
 339         if (DEV_SUPERSPEED(reg))
 340                 return USB_SPEED_SUPER;
 341         else if (DEV_HIGHSPEED(reg))
 342                 return USB_SPEED_HIGH;
 343         else if (DEV_FULLSPEED(reg))
 344                 return USB_SPEED_FULL;
 345         else if (DEV_LOWSPEED(reg))
 346                 return USB_SPEED_LOW;
 347         return USB_SPEED_UNKNOWN;
 348 }
 349
 350 /**
 351  * cdns3_start_all_request - add to ring all request not started
 352  * @priv_dev: Extended gadget object
 353  * @priv_ep: The endpoint for whom request will be started.
 354  *
 355  * Returns return ENOMEM if transfer ring i not enough TRBs to start
 356  *         all requests.
 357  */
 358 static int cdns3_start_all_request(struct cdns3_device *priv_dev,
 359                                    struct cdns3_endpoint *priv_ep)
 360 {
 361         struct usb_request *request;
 362         int ret = 0;
 363
 364         while (!list_empty(&priv_ep->deferred_req_list)) {
 365                 request = cdns3_next_request(&priv_ep->deferred_req_list);
 366
 367                 ret = cdns3_ep_run_transfer(priv_ep, request);
 368                 if (ret)
 369                         return ret;
 370
 371                 list_del(&request->list);
 372                 list_add_tail(&request->list,
 373                               &priv_ep->pending_req_list);
 374         }
 375
 376         priv_ep->flags &= ~EP_RING_FULL;
 377         return ret;
 378 }
 379
 380 /*
 381  * WA2: Set flag for all not ISOC OUT endpoints. If this flag is set
 382  * driver try to detect whether endpoint need additional internal
 383  * buffer for unblocking on-chip FIFO buffer. This flag will be cleared
 384  * if before first DESCMISS interrupt the DMA will be armed.
 385  */
 386 #define cdns3_wa2_enable_detection(priv_dev, ep_priv, reg) do { \
 387         if (!priv_ep->dir && priv_ep->type != USB_ENDPOINT_XFER_ISOC) { \
 388                 priv_ep->flags |= EP_QUIRK_EXTRA_BUF_DET; \
 389                 (reg) |= EP_STS_EN_DESCMISEN; \
 390         } } while (0)
 391
 392 /**
 393  * cdns3_wa2_descmiss_copy_data copy data from internal requests to
 394  * request queued by class driver.
 395  * @priv_ep: extended endpoint object
 396  * @request: request object
 397  */
 398 static void cdns3_wa2_descmiss_copy_data(struct cdns3_endpoint *priv_ep,
 399                                          struct usb_request *request)
 400 {
 401         struct usb_request *descmiss_req;
 402         struct cdns3_request *descmiss_priv_req;
 403
 404         while (!list_empty(&priv_ep->wa2_descmiss_req_list)) {
 405                 int chunk_end;
 406                 int length;
 407
 408                 descmiss_priv_req =
 409                         cdns3_next_priv_request(&priv_ep->wa2_descmiss_req_list);
 410                 descmiss_req = &descmiss_priv_req->request;
 411
 412                 /* driver can't touch pending request */
 413                 if (descmiss_priv_req->flags & REQUEST_PENDING)
 414                         break;
 415
 416                 chunk_end = descmiss_priv_req->flags & REQUEST_INTERNAL_CH;
 417                 length = request->actual + descmiss_req->actual;
 418
 419                 request->status = descmiss_req->status;
 420
 421                 if (length <= request->length) {
 422                         memcpy(&((u8 *)request->buf)[request->actual],
 423                                descmiss_req->buf,
 424                                descmiss_req->actual);
 425                         request->actual = length;
 426                 } else {
 427                         /* It should never occur */
 428                         request->status = -ENOMEM;
 429                 }
 430
 431                 list_del_init(&descmiss_priv_req->list);
 432
 433                 kfree(descmiss_req->buf);
 434                 cdns3_gadget_ep_free_request(&priv_ep->endpoint, descmiss_req);
 435                 --priv_ep->wa2_counter;
 436
 437                 if (!chunk_end)
 438                         break;
 439         }
 440 }
 441
 442 struct usb_request *cdns3_wa2_gadget_giveback(struct cdns3_device *priv_dev,
 443                                               struct cdns3_endpoint *priv_ep,
 444                                               struct cdns3_request *priv_req)
 445 {
 446         if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN &&
 447             priv_req->flags & REQUEST_INTERNAL) {
 448                 struct usb_request *req;
 449
 450                 req = cdns3_next_request(&priv_ep->deferred_req_list);
 451
 452                 priv_ep->descmis_req = NULL;
 453
 454                 if (!req)
 455                         return NULL;
 456
 457                 cdns3_wa2_descmiss_copy_data(priv_ep, req);
 458                 if (!(priv_ep->flags & EP_QUIRK_END_TRANSFER) &&
 459                     req->length != req->actual) {
 460                         /* wait for next part of transfer */
 461                         return NULL;
 462                 }
 463
 464                 if (req->status == -EINPROGRESS)
 465                         req->status = 0;
 466
 467                 list_del_init(&req->list);
 468                 cdns3_start_all_request(priv_dev, priv_ep);
 469                 return req;
 470         }
 471
 472         return &priv_req->request;
 473 }
 474
 475 int cdns3_wa2_gadget_ep_queue(struct cdns3_device *priv_dev,
 476                               struct cdns3_endpoint *priv_ep,
 477                               struct cdns3_request *priv_req)
 478 {
 479         int deferred = 0;
 480
 481         /*
 482          * If transfer was queued before DESCMISS appear than we
 483          * can disable handling of DESCMISS interrupt. Driver assumes that it
 484          * can disable special treatment for this endpoint.
 485          */
 486         if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_DET) {
 487                 u32 reg;
 488
 489                 cdns3_select_ep(priv_dev, priv_ep->num | priv_ep->dir);
 490                 priv_ep->flags &= ~EP_QUIRK_EXTRA_BUF_DET;
 491                 reg = readl(&priv_dev->regs->ep_sts_en);
 492                 reg &= ~EP_STS_EN_DESCMISEN;
 493                 trace_cdns3_wa2(priv_ep, "workaround disabled\n");
 494                 writel(reg, &priv_dev->regs->ep_sts_en);
 495         }
 496
 497         if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN) {
 498                 u8 pending_empty = list_empty(&priv_ep->pending_req_list);
 499                 u8 descmiss_empty = list_empty(&priv_ep->wa2_descmiss_req_list);
 500
 501                 /*
 502                  *  DESCMISS transfer has been finished, so data will be
 503                  *  directly copied from internal allocated usb_request
 504                  *  objects.
 505                  */
 506                 if (pending_empty && !descmiss_empty &&
 507                     !(priv_req->flags & REQUEST_INTERNAL)) {
 508                         cdns3_wa2_descmiss_copy_data(priv_ep,
 509                                                      &priv_req->request);
 510
 511                         trace_cdns3_wa2(priv_ep, "get internal stored data");
 512
 513                         list_add_tail(&priv_req->request.list,
 514                                       &priv_ep->pending_req_list);
 515                         cdns3_gadget_giveback(priv_ep, priv_req,
 516                                               priv_req->request.status);
 517
 518                         /*
 519                          * Intentionally driver returns positive value as
 520                          * correct value. It informs that transfer has
 521                          * been finished.
 522                          */
 523                         return EINPROGRESS;
 524                 }
 525
 526                 /*
 527                  * Driver will wait for completion DESCMISS transfer,
 528                  * before starts new, not DESCMISS transfer.
 529                  */
 530                 if (!pending_empty && !descmiss_empty) {
 531                         trace_cdns3_wa2(priv_ep, "wait for pending transfer\n");
 532                         deferred = 1;
 533                 }
 534
 535                 if (priv_req->flags & REQUEST_INTERNAL)
 536                         list_add_tail(&priv_req->list,
 537                                       &priv_ep->wa2_descmiss_req_list);
 538         }
 539
 540         return deferred;
 541 }
 542
 543 static void cdns3_wa2_remove_old_request(struct cdns3_endpoint *priv_ep)
 544 {
 545         struct cdns3_request *priv_req;
 546
 547         while (!list_empty(&priv_ep->wa2_descmiss_req_list)) {
 548                 u8 chain;
 549
 550                 priv_req = cdns3_next_priv_request(&priv_ep->wa2_descmiss_req_list);
 551                 chain = !!(priv_req->flags & REQUEST_INTERNAL_CH);
 552
 553                 trace_cdns3_wa2(priv_ep, "removes eldest request");
 554
 555                 kfree(priv_req->request.buf);
 556                 cdns3_gadget_ep_free_request(&priv_ep->endpoint,
 557                                              &priv_req->request);
 558                 list_del_init(&priv_req->list);
 559                 --priv_ep->wa2_counter;
 560
 561                 if (!chain)
 562                         break;
 563         }
 564 }
 565
 566 /**
 567  * cdns3_wa2_descmissing_packet - handles descriptor missing event.
 568  * @priv_dev: extended gadget object
 569  *
 570  * This function is used only for WA2. For more information see Work around 2
 571  * description.
 572  */
 573 static void cdns3_wa2_descmissing_packet(struct cdns3_endpoint *priv_ep)
 574 {
 575         struct cdns3_request *priv_req;
 576         struct usb_request *request;
 577
 578         if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_DET) {
 579                 priv_ep->flags &= ~EP_QUIRK_EXTRA_BUF_DET;
 580                 priv_ep->flags |= EP_QUIRK_EXTRA_BUF_EN;
 581         }
 582
 583         trace_cdns3_wa2(priv_ep, "Description Missing detected\n");
 584
 585         if (priv_ep->wa2_counter >= CDNS3_WA2_NUM_BUFFERS)
 586                 cdns3_wa2_remove_old_request(priv_ep);
 587
 588         request = cdns3_gadget_ep_alloc_request(&priv_ep->endpoint,
 589                                                 GFP_ATOMIC);
 590         if (!request)
 591                 goto err;
 592
 593         priv_req = to_cdns3_request(request);
 594         priv_req->flags |= REQUEST_INTERNAL;
 595
 596         /* if this field is still assigned it indicate that transfer related
 597          * with this request has not been finished yet. Driver in this
 598          * case simply allocate next request and assign flag REQUEST_INTERNAL_CH
 599          * flag to previous one. It will indicate that current request is
 600          * part of the previous one.
 601          */
 602         if (priv_ep->descmis_req)
 603                 priv_ep->descmis_req->flags |= REQUEST_INTERNAL_CH;
 604
 605         priv_req->request.buf = kzalloc(CDNS3_DESCMIS_BUF_SIZE,
 606                                         GFP_ATOMIC);
 607         priv_ep->wa2_counter++;
 608
 609         if (!priv_req->request.buf) {
 610                 cdns3_gadget_ep_free_request(&priv_ep->endpoint, request);
 611                 goto err;
 612         }
 613
 614         priv_req->request.length = CDNS3_DESCMIS_BUF_SIZE;
 615         priv_ep->descmis_req = priv_req;
 616
 617         __cdns3_gadget_ep_queue(&priv_ep->endpoint,
 618                                 &priv_ep->descmis_req->request,
 619                                 GFP_ATOMIC);
 620
 621         return;
 622
 623 err:
 624         dev_err(priv_ep->cdns3_dev->dev,
 625                 "Failed: No sufficient memory for DESCMIS\n");
 626 }
 627
 628 /**
 629  * cdns3_gadget_giveback - call struct usb_request's ->complete callback
 630  * @priv_ep: The endpoint to whom the request belongs to
 631  * @priv_req: The request we're giving back
 632  * @status: completion code for the request
 633  *
 634  * Must be called with controller's lock held and interrupts disabled. This
 635  * function will unmap @req and call its ->complete() callback to notify upper
 636  * layers that it has completed.
 637  */
 638 void cdns3_gadget_giveback(struct cdns3_endpoint *priv_ep,
 639                            struct cdns3_request *priv_req,
 640                            int status)
 641 {
 642         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
 643         struct usb_request *request = &priv_req->request;
 644
 645         list_del_init(&request->list);
 646
 647         if (request->status == -EINPROGRESS)
 648                 request->status = status;
 649
 650         usb_gadget_unmap_request(&priv_dev->gadget, request,
 651                                  priv_ep->dir);
 652
 653         if ((priv_req->flags & REQUEST_UNALIGNED) &&
 654             priv_ep->dir == USB_DIR_OUT && !request->status)
 655                 memcpy(request->buf, priv_req->aligned_buf->buf,
 656                        request->length);
 657
 658         priv_req->flags &= ~(REQUEST_PENDING | REQUEST_UNALIGNED);
 659         trace_cdns3_gadget_giveback(priv_req);
 660
 661         if (priv_dev->dev_ver < DEV_VER_V2) {
 662                 request = cdns3_wa2_gadget_giveback(priv_dev, priv_ep,
 663                                                     priv_req);
 664                 if (!request)
 665                         return;
 666         }
 667
 668         if (request->complete) {
 669                 spin_unlock(&priv_dev->lock);
 670                 usb_gadget_giveback_request(&priv_ep->endpoint,
 671                                             request);
 672                 spin_lock(&priv_dev->lock);
 673         }
 674
 675         if (request->buf == priv_dev->zlp_buf)
 676                 cdns3_gadget_ep_free_request(&priv_ep->endpoint, request);
 677 }
 678
 679 void cdns3_wa1_restore_cycle_bit(struct cdns3_endpoint *priv_ep)
 680 {
 681         /* Work around for stale data address in TRB*/
 682         if (priv_ep->wa1_set) {
 683                 trace_cdns3_wa1(priv_ep, "restore cycle bit");
 684
 685                 priv_ep->wa1_set = 0;
 686                 priv_ep->wa1_trb_index = 0xFFFF;
 687                 if (priv_ep->wa1_cycle_bit) {
 688                         priv_ep->wa1_trb->control =
 689                                 priv_ep->wa1_trb->control | 0x1;
 690                 } else {
 691                         priv_ep->wa1_trb->control =
 692                                 priv_ep->wa1_trb->control & ~0x1;
 693                 }
 694         }
 695 }
 696
 697 static void cdns3_free_aligned_request_buf(struct cdns3_device *priv_dev)
 698 {
 699         struct cdns3_aligned_buf *buf, *tmp;
 700         unsigned long flags;
 701
 702         spin_lock_irqsave(&priv_dev->lock, flags);
 703
 704         list_for_each_entry_safe(buf, tmp, &priv_dev->aligned_buf_list, list) {
 705                 if (!buf->in_use) {
 706                         list_del(&buf->list);
 707
 708                         /*
 709                          * Re-enable interrupts to free DMA capable memory.
 710                          * Driver can't free this memory with disabled
 711                          * interrupts.
 712                          */
 713                         spin_unlock_irqrestore(&priv_dev->lock, flags);
 714                         dma_free_coherent(buf->buf);
 715                         kfree(buf);
 716                         spin_lock_irqsave(&priv_dev->lock, flags);
 717                 }
 718         }
 719
 720         spin_unlock_irqrestore(&priv_dev->lock, flags);
 721 }
 722
 723 static int cdns3_prepare_aligned_request_buf(struct cdns3_request *priv_req)
 724 {
 725         struct cdns3_endpoint *priv_ep = priv_req->priv_ep;
 726         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
 727         struct cdns3_aligned_buf *buf;
 728
 729         /* check if buffer is aligned to 8. */
 730         if (!((uintptr_t)priv_req->request.buf & 0x7))
 731                 return 0;
 732
 733         buf = priv_req->aligned_buf;
 734
 735         if (!buf || priv_req->request.length > buf->size) {
 736                 buf = kzalloc(sizeof(*buf), GFP_ATOMIC);
 737                 if (!buf)
 738                         return -ENOMEM;
 739
 740                 buf->size = priv_req->request.length;
 741
 742                 buf->buf = dma_alloc_coherent(buf->size,
 743                                               (unsigned long *)&buf->dma);
 744                 if (!buf->buf) {
 745                         kfree(buf);
 746                         return -ENOMEM;
 747                 }
 748
 749                 if (priv_req->aligned_buf) {
 750                         trace_cdns3_free_aligned_request(priv_req);
 751                         priv_req->aligned_buf->in_use = 0;
 752 #ifndef __UBOOT__
 753                         queue_work(system_freezable_wq,
 754                                    &priv_dev->aligned_buf_wq);
 755 #else
 756                         cdns3_free_aligned_request_buf(priv_dev);
 757 #endif
 758                 }
 759
 760                 buf->in_use = 1;
 761                 priv_req->aligned_buf = buf;
 762
 763                 list_add_tail(&buf->list,
 764                               &priv_dev->aligned_buf_list);
 765         }
 766
 767         if (priv_ep->dir == USB_DIR_IN) {
 768                 memcpy(buf->buf, priv_req->request.buf,
 769                        priv_req->request.length);
 770         }
 771
 772         priv_req->flags |= REQUEST_UNALIGNED;
 773         trace_cdns3_prepare_aligned_request(priv_req);
 774
 775         return 0;
 776 }
 777
 778 static int cdns3_wa1_update_guard(struct cdns3_endpoint *priv_ep,
 779                                   struct cdns3_trb *trb)
 780 {
 781         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
 782
 783         if (!priv_ep->wa1_set) {
 784                 u32 doorbell;
 785
 786                 doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY);
 787
 788                 if (doorbell) {
 789                         priv_ep->wa1_cycle_bit = priv_ep->pcs ? TRB_CYCLE : 0;
 790                         priv_ep->wa1_set = 1;
 791                         priv_ep->wa1_trb = trb;
 792                         priv_ep->wa1_trb_index = priv_ep->enqueue;
 793                         trace_cdns3_wa1(priv_ep, "set guard");
 794                         return 0;
 795                 }
 796         }
 797         return 1;
 798 }
 799
 800 static void cdns3_wa1_tray_restore_cycle_bit(struct cdns3_device *priv_dev,
 801                                              struct cdns3_endpoint *priv_ep)
 802 {
 803         int dma_index;
 804         u32 doorbell;
 805
 806         doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY);
 807         dma_index = cdns3_get_dma_pos(priv_dev, priv_ep);
 808
 809         if (!doorbell || dma_index != priv_ep->wa1_trb_index)
 810                 cdns3_wa1_restore_cycle_bit(priv_ep);
 811 }
 812
 813 /**
 814  * cdns3_ep_run_transfer - start transfer on no-default endpoint hardware
 815  * @priv_ep: endpoint object
 816  *
 817  * Returns zero on success or negative value on failure
 818  */
 819 int cdns3_ep_run_transfer(struct cdns3_endpoint *priv_ep,
 820                           struct usb_request *request)
 821 {
 822         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
 823         struct cdns3_request *priv_req;
 824         struct cdns3_trb *trb;
 825         dma_addr_t trb_dma;
 826         u32 togle_pcs = 1;
 827         int sg_iter = 0;
 828         int num_trb = 1;
 829         int address;
 830         u32 control;
 831         int pcs;
 832
 833         if (num_trb > priv_ep->free_trbs) {
 834                 priv_ep->flags |= EP_RING_FULL;
 835                 return -ENOBUFS;
 836         }
 837
 838         priv_req = to_cdns3_request(request);
 839         address = priv_ep->endpoint.desc->bEndpointAddress;
 840
 841         priv_ep->flags |= EP_PENDING_REQUEST;
 842
 843         /* must allocate buffer aligned to 8 */
 844         if (priv_req->flags & REQUEST_UNALIGNED)
 845                 trb_dma = priv_req->aligned_buf->dma;
 846         else
 847                 trb_dma = request->dma;
 848
 849         trb = priv_ep->trb_pool + priv_ep->enqueue;
 850         priv_req->start_trb = priv_ep->enqueue;
 851         priv_req->trb = trb;
 852
 853         cdns3_select_ep(priv_ep->cdns3_dev, address);
 854
 855         /* prepare ring */
 856         if ((priv_ep->enqueue + num_trb)  >= (priv_ep->num_trbs - 1)) {
 857                 struct cdns3_trb *link_trb;
 858                 int doorbell, dma_index;
 859                 u32 ch_bit = 0;
 860
 861                 doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY);
 862                 dma_index = cdns3_get_dma_pos(priv_dev, priv_ep);
 863
 864                 /* Driver can't update LINK TRB if it is current processed. */
 865                 if (doorbell && dma_index == priv_ep->num_trbs - 1) {
 866                         priv_ep->flags |= EP_DEFERRED_DRDY;
 867                         return -ENOBUFS;
 868                 }
 869
 870                 /*updating C bt in  Link TRB before starting DMA*/
 871                 link_trb = priv_ep->trb_pool + (priv_ep->num_trbs - 1);
 872                 /*
 873                  * For TRs size equal 2 enabling TRB_CHAIN for epXin causes
 874                  * that DMA stuck at the LINK TRB.
 875                  * On the other hand, removing TRB_CHAIN for longer TRs for
 876                  * epXout cause that DMA stuck after handling LINK TRB.
 877                  * To eliminate this strange behavioral driver set TRB_CHAIN
 878                  * bit only for TR size > 2.
 879                  */
 880                 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC ||
 881                     TRBS_PER_SEGMENT > 2)
 882                         ch_bit = TRB_CHAIN;
 883
 884                 link_trb->control = ((priv_ep->pcs) ? TRB_CYCLE : 0) |
 885                                     TRB_TYPE(TRB_LINK) | TRB_TOGGLE | ch_bit;
 886         }
 887
 888         if (priv_dev->dev_ver <= DEV_VER_V2)
 889                 togle_pcs = cdns3_wa1_update_guard(priv_ep, trb);
 890
 891         /* set incorrect Cycle Bit for first trb*/
 892         control = priv_ep->pcs ? 0 : TRB_CYCLE;
 893
 894         do {
 895                 u32 length;
 896                 u16 td_size = 0;
 897
 898                 /* fill TRB */
 899                 control |= TRB_TYPE(TRB_NORMAL);
 900                 trb->buffer = TRB_BUFFER(trb_dma);
 901
 902                 length = request->length;
 903
 904                 if (likely(priv_dev->dev_ver >= DEV_VER_V2))
 905                         td_size = DIV_ROUND_UP(length,
 906                                                priv_ep->endpoint.maxpacket);
 907
 908                 trb->length = TRB_BURST_LEN(priv_ep->trb_burst_size) |
 909                                         TRB_LEN(length);
 910                 if (priv_dev->gadget.speed == USB_SPEED_SUPER)
 911                         trb->length |= TRB_TDL_SS_SIZE(td_size);
 912                 else
 913                         control |= TRB_TDL_HS_SIZE(td_size);
 914
 915                 pcs = priv_ep->pcs ? TRB_CYCLE : 0;
 916
 917                 /*
 918                  * first trb should be prepared as last to avoid processing
 919                  *  transfer to early
 920                  */
 921                 if (sg_iter != 0)
 922                         control |= pcs;
 923
 924                 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC  && !priv_ep->dir) {
 925                         control |= TRB_IOC | TRB_ISP;
 926                 } else {
 927                         /* for last element in TD or in SG list */
 928                         if (sg_iter == (num_trb - 1) && sg_iter != 0)
 929                                 control |= pcs | TRB_IOC | TRB_ISP;
 930                 }
 931
 932                 if (sg_iter)
 933                         trb->control = control;
 934                 else
 935                         priv_req->trb->control = control;
 936
 937                 control = 0;
 938                 ++sg_iter;
 939                 priv_req->end_trb = priv_ep->enqueue;
 940                 cdns3_ep_inc_enq(priv_ep);
 941                 trb = priv_ep->trb_pool + priv_ep->enqueue;
 942         } while (sg_iter < num_trb);
 943
 944         trb = priv_req->trb;
 945
 946         priv_req->flags |= REQUEST_PENDING;
 947
 948         if (sg_iter == 1)
 949                 trb->control |= TRB_IOC | TRB_ISP;
 950
 951         /*
 952          * Memory barrier - cycle bit must be set before other filds in trb.
 953          */
 954         dmb();
 955
 956         /* give the TD to the consumer*/
 957         if (togle_pcs)
 958                 trb->control =  trb->control ^ 1;
 959
 960         if (priv_dev->dev_ver <= DEV_VER_V2)
 961                 cdns3_wa1_tray_restore_cycle_bit(priv_dev, priv_ep);
 962
 963         trace_cdns3_prepare_trb(priv_ep, priv_req->trb);
 964
 965         /*
 966          * Memory barrier - Cycle Bit must be set before trb->length  and
 967          * trb->buffer fields.
 968          */
 969         dmb();
 970
 971         /*
 972          * For DMULT mode we can set address to transfer ring only once after
 973          * enabling endpoint.
 974          */
 975         if (priv_ep->flags & EP_UPDATE_EP_TRBADDR) {
 976                 /*
 977                  * Until SW is not ready to handle the OUT transfer the ISO OUT
 978                  * Endpoint should be disabled (EP_CFG.ENABLE = 0).
 979                  * EP_CFG_ENABLE must be set before updating ep_traddr.
 980                  */
 981                 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC  && !priv_ep->dir &&
 982                     !(priv_ep->flags & EP_QUIRK_ISO_OUT_EN)) {
 983                         priv_ep->flags |= EP_QUIRK_ISO_OUT_EN;
 984                         cdns3_set_register_bit(&priv_dev->regs->ep_cfg,
 985                                                EP_CFG_ENABLE);
 986                 }
 987
 988                 writel(EP_TRADDR_TRADDR(priv_ep->trb_pool_dma +
 989                                         priv_req->start_trb * TRB_SIZE),
 990                                         &priv_dev->regs->ep_traddr);
 991
 992                 priv_ep->flags &= ~EP_UPDATE_EP_TRBADDR;
 993         }
 994
 995         if (!priv_ep->wa1_set && !(priv_ep->flags & EP_STALLED)) {
 996                 trace_cdns3_ring(priv_ep);
 997                 /*clearing TRBERR and EP_STS_DESCMIS before seting DRDY*/
 998                 writel(EP_STS_TRBERR | EP_STS_DESCMIS, &priv_dev->regs->ep_sts);
 999                 writel(EP_CMD_DRDY, &priv_dev->regs->ep_cmd);
1000                 trace_cdns3_doorbell_epx(priv_ep->name,
1001                                          readl(&priv_dev->regs->ep_traddr));
1002         }
1003
1004         /* WORKAROUND for transition to L0 */
1005         __cdns3_gadget_wakeup(priv_dev);
1006
1007         return 0;
1008 }
1009
1010 void cdns3_set_hw_configuration(struct cdns3_device *priv_dev)
1011 {
1012         struct cdns3_endpoint *priv_ep;
1013         struct usb_ep *ep;
1014         int val;
1015
1016         if (priv_dev->hw_configured_flag)
1017                 return;
1018
1019         writel(USB_CONF_CFGSET, &priv_dev->regs->usb_conf);
1020         writel(EP_CMD_ERDY | EP_CMD_REQ_CMPL, &priv_dev->regs->ep_cmd);
1021
1022         cdns3_set_register_bit(&priv_dev->regs->usb_conf,
1023                                USB_CONF_U1EN | USB_CONF_U2EN);
1024
1025         /* wait until configuration set */
1026         readl_poll_timeout_atomic(&priv_dev->regs->usb_sts, val,
1027                                   val & USB_STS_CFGSTS_MASK, 100);
1028
1029         priv_dev->hw_configured_flag = 1;
1030
1031         list_for_each_entry(ep, &priv_dev->gadget.ep_list, ep_list) {
1032                 priv_ep = ep_to_cdns3_ep(ep);
1033                 if (priv_ep->flags & EP_ENABLED)
1034                         cdns3_start_all_request(priv_dev, priv_ep);
1035         }
1036 }
1037
1038 /**
1039  * cdns3_request_handled - check whether request has been handled by DMA
1040  *
1041  * @priv_ep: extended endpoint object.
1042  * @priv_req: request object for checking
1043  *
1044  * Endpoint must be selected before invoking this function.
1045  *
1046  * Returns false if request has not been handled by DMA, else returns true.
1047  *
1048  * SR - start ring
1049  * ER -  end ring
1050  * DQ = priv_ep->dequeue - dequeue position
1051  * EQ = priv_ep->enqueue -  enqueue position
1052  * ST = priv_req->start_trb - index of first TRB in transfer ring
1053  * ET = priv_req->end_trb - index of last TRB in transfer ring
1054  * CI = current_index - index of processed TRB by DMA.
1055  *
1056  * As first step, function checks if cycle bit for priv_req->start_trb is
1057  * correct.
1058  *
1059  * some rules:
1060  * 1. priv_ep->dequeue never exceed current_index.
1061  * 2  priv_ep->enqueue never exceed priv_ep->dequeue
1062  * 3. exception: priv_ep->enqueue == priv_ep->dequeue
1063  *    and priv_ep->free_trbs is zero.
1064  *    This case indicate that TR is full.
1065  *
1066  * Then We can split recognition into two parts:
1067  * Case 1 - priv_ep->dequeue < current_index
1068  *      SR ... EQ ... DQ ... CI ... ER
1069  *      SR ... DQ ... CI ... EQ ... ER
1070  *
1071  *      Request has been handled by DMA if ST and ET is between DQ and CI.
1072  *
1073  * Case 2 - priv_ep->dequeue > current_index
1074  * This situation take place when CI go through the LINK TRB at the end of
1075  * transfer ring.
1076  *      SR ... CI ... EQ ... DQ ... ER
1077  *
1078  *      Request has been handled by DMA if ET is less then CI or
1079  *      ET is greater or equal DQ.
1080  */
1081 static bool cdns3_request_handled(struct cdns3_endpoint *priv_ep,
1082                                   struct cdns3_request *priv_req)
1083 {
1084         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1085         struct cdns3_trb *trb = priv_req->trb;
1086         int current_index = 0;
1087         int handled = 0;
1088         int doorbell;
1089
1090         current_index = cdns3_get_dma_pos(priv_dev, priv_ep);
1091         doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY);
1092
1093         trb = &priv_ep->trb_pool[priv_req->start_trb];
1094
1095         if ((trb->control  & TRB_CYCLE) != priv_ep->ccs)
1096                 goto finish;
1097
1098         if (doorbell == 1 && current_index == priv_ep->dequeue)
1099                 goto finish;
1100
1101         /* The corner case for TRBS_PER_SEGMENT equal 2). */
1102         if (TRBS_PER_SEGMENT == 2 && priv_ep->type != USB_ENDPOINT_XFER_ISOC) {
1103                 handled = 1;
1104                 goto finish;
1105         }
1106
1107         if (priv_ep->enqueue == priv_ep->dequeue &&
1108             priv_ep->free_trbs == 0) {
1109                 handled = 1;
1110         } else if (priv_ep->dequeue < current_index) {
1111                 if ((current_index == (priv_ep->num_trbs - 1)) &&
1112                     !priv_ep->dequeue)
1113                         goto finish;
1114
1115                 if (priv_req->end_trb >= priv_ep->dequeue &&
1116                     priv_req->end_trb < current_index)
1117                         handled = 1;
1118         } else if (priv_ep->dequeue  > current_index) {
1119                 if (priv_req->end_trb  < current_index ||
1120                     priv_req->end_trb >= priv_ep->dequeue)
1121                         handled = 1;
1122         }
1123
1124 finish:
1125         trace_cdns3_request_handled(priv_req, current_index, handled);
1126
1127         return handled;
1128 }
1129
1130 static void cdns3_transfer_completed(struct cdns3_device *priv_dev,
1131                                      struct cdns3_endpoint *priv_ep)
1132 {
1133         struct cdns3_request *priv_req;
1134         struct usb_request *request;
1135         struct cdns3_trb *trb;
1136
1137         while (!list_empty(&priv_ep->pending_req_list)) {
1138                 request = cdns3_next_request(&priv_ep->pending_req_list);
1139                 priv_req = to_cdns3_request(request);
1140
1141                 /* Re-select endpoint. It could be changed by other CPU during
1142                  * handling usb_gadget_giveback_request.
1143                  */
1144 #ifndef __UBOOT__
1145                 cdns3_select_ep(priv_dev, priv_ep->endpoint.address);
1146 #else
1147                 cdns3_select_ep(priv_dev,
1148                                 priv_ep->endpoint.desc->bEndpointAddress);
1149 #endif
1150
1151                 if (!cdns3_request_handled(priv_ep, priv_req))
1152                         goto prepare_next_td;
1153
1154                 trb = priv_ep->trb_pool + priv_ep->dequeue;
1155                 trace_cdns3_complete_trb(priv_ep, trb);
1156
1157                 if (trb != priv_req->trb)
1158                         dev_warn(priv_dev->dev,
1159                                  "request_trb=0x%p, queue_trb=0x%p\n",
1160                                  priv_req->trb, trb);
1161
1162                 request->actual = TRB_LEN(le32_to_cpu(trb->length));
1163                 cdns3_move_deq_to_next_trb(priv_req);
1164                 cdns3_gadget_giveback(priv_ep, priv_req, 0);
1165
1166                 if (priv_ep->type != USB_ENDPOINT_XFER_ISOC &&
1167                     TRBS_PER_SEGMENT == 2)
1168                         break;
1169         }
1170         priv_ep->flags &= ~EP_PENDING_REQUEST;
1171
1172 prepare_next_td:
1173         if (!(priv_ep->flags & EP_STALLED) &&
1174             !(priv_ep->flags & EP_STALL_PENDING))
1175                 cdns3_start_all_request(priv_dev, priv_ep);
1176 }
1177
1178 void cdns3_rearm_transfer(struct cdns3_endpoint *priv_ep, u8 rearm)
1179 {
1180         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1181
1182         cdns3_wa1_restore_cycle_bit(priv_ep);
1183
1184         if (rearm) {
1185                 trace_cdns3_ring(priv_ep);
1186
1187                 /* Cycle Bit must be updated before arming DMA. */
1188                 dmb();
1189                 writel(EP_CMD_DRDY, &priv_dev->regs->ep_cmd);
1190
1191                 __cdns3_gadget_wakeup(priv_dev);
1192
1193                 trace_cdns3_doorbell_epx(priv_ep->name,
1194                                          readl(&priv_dev->regs->ep_traddr));
1195         }
1196 }
1197
1198 /**
1199  * cdns3_check_ep_interrupt_proceed - Processes interrupt related to endpoint
1200  * @priv_ep: endpoint object
1201  *
1202  * Returns 0
1203  */
1204 static int cdns3_check_ep_interrupt_proceed(struct cdns3_endpoint *priv_ep)
1205 {
1206         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1207         u32 ep_sts_reg;
1208
1209 #ifndef __UBOOT__
1210         cdns3_select_ep(priv_dev, priv_ep->endpoint.address);
1211 #else
1212         cdns3_select_ep(priv_dev, priv_ep->endpoint.desc->bEndpointAddress);
1213 #endif
1214
1215         trace_cdns3_epx_irq(priv_dev, priv_ep);
1216
1217         ep_sts_reg = readl(&priv_dev->regs->ep_sts);
1218         writel(ep_sts_reg, &priv_dev->regs->ep_sts);
1219
1220         if (ep_sts_reg & EP_STS_TRBERR) {
1221                 if (priv_ep->flags & EP_STALL_PENDING &&
1222                     !(ep_sts_reg & EP_STS_DESCMIS &&
1223                     priv_dev->dev_ver < DEV_VER_V2)) {
1224                         cdns3_ep_stall_flush(priv_ep);
1225                 }
1226
1227                 /*
1228                  * For isochronous transfer driver completes request on
1229                  * IOC or on TRBERR. IOC appears only when device receive
1230                  * OUT data packet. If host disable stream or lost some packet
1231                  * then the only way to finish all queued transfer is to do it
1232                  * on TRBERR event.
1233                  */
1234                 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC &&
1235                     !priv_ep->wa1_set) {
1236                         if (!priv_ep->dir) {
1237                                 u32 ep_cfg = readl(&priv_dev->regs->ep_cfg);
1238
1239                                 ep_cfg &= ~EP_CFG_ENABLE;
1240                                 writel(ep_cfg, &priv_dev->regs->ep_cfg);
1241                                 priv_ep->flags &= ~EP_QUIRK_ISO_OUT_EN;
1242                         }
1243                         cdns3_transfer_completed(priv_dev, priv_ep);
1244                 } else if (!(priv_ep->flags & EP_STALLED) &&
1245                           !(priv_ep->flags & EP_STALL_PENDING)) {
1246                         if (priv_ep->flags & EP_DEFERRED_DRDY) {
1247                                 priv_ep->flags &= ~EP_DEFERRED_DRDY;
1248                                 cdns3_start_all_request(priv_dev, priv_ep);
1249                         } else {
1250                                 cdns3_rearm_transfer(priv_ep,
1251                                                      priv_ep->wa1_set);
1252                         }
1253                 }
1254         }
1255
1256         if ((ep_sts_reg & EP_STS_IOC) || (ep_sts_reg & EP_STS_ISP)) {
1257                 if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN) {
1258                         if (ep_sts_reg & EP_STS_ISP)
1259                                 priv_ep->flags |= EP_QUIRK_END_TRANSFER;
1260                         else
1261                                 priv_ep->flags &= ~EP_QUIRK_END_TRANSFER;
1262                 }
1263
1264                 cdns3_transfer_completed(priv_dev, priv_ep);
1265         }
1266
1267         /*
1268          * WA2: this condition should only be meet when
1269          * priv_ep->flags & EP_QUIRK_EXTRA_BUF_DET or
1270          * priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN.
1271          * In other cases this interrupt will be disabled/
1272          */
1273         if (ep_sts_reg & EP_STS_DESCMIS && priv_dev->dev_ver < DEV_VER_V2 &&
1274             !(priv_ep->flags & EP_STALLED))
1275                 cdns3_wa2_descmissing_packet(priv_ep);
1276
1277         return 0;
1278 }
1279
1280 static void cdns3_disconnect_gadget(struct cdns3_device *priv_dev)
1281 {
1282         if (priv_dev->gadget_driver && priv_dev->gadget_driver->disconnect) {
1283                 spin_unlock(&priv_dev->lock);
1284                 priv_dev->gadget_driver->disconnect(&priv_dev->gadget);
1285                 spin_lock(&priv_dev->lock);
1286         }
1287 }
1288
1289 /**
1290  * cdns3_check_usb_interrupt_proceed - Processes interrupt related to device
1291  * @priv_dev: extended gadget object
1292  * @usb_ists: bitmap representation of device's reported interrupts
1293  * (usb_ists register value)
1294  */
1295 static void cdns3_check_usb_interrupt_proceed(struct cdns3_device *priv_dev,
1296                                               u32 usb_ists)
1297 {
1298         int speed = 0;
1299
1300         trace_cdns3_usb_irq(priv_dev, usb_ists);
1301         if (usb_ists & USB_ISTS_L1ENTI) {
1302                 /*
1303                  * WORKAROUND: CDNS3 controller has issue with hardware resuming
1304                  * from L1. To fix it, if any DMA transfer is pending driver
1305                  * must starts driving resume signal immediately.
1306                  */
1307                 if (readl(&priv_dev->regs->drbl))
1308                         __cdns3_gadget_wakeup(priv_dev);
1309         }
1310
1311         /* Connection detected */
1312         if (usb_ists & (USB_ISTS_CON2I | USB_ISTS_CONI)) {
1313                 speed = cdns3_get_speed(priv_dev);
1314                 priv_dev->gadget.speed = speed;
1315                 usb_gadget_set_state(&priv_dev->gadget, USB_STATE_POWERED);
1316                 cdns3_ep0_config(priv_dev);
1317         }
1318
1319         /* Disconnection detected */
1320         if (usb_ists & (USB_ISTS_DIS2I | USB_ISTS_DISI)) {
1321                 cdns3_disconnect_gadget(priv_dev);
1322                 priv_dev->gadget.speed = USB_SPEED_UNKNOWN;
1323                 usb_gadget_set_state(&priv_dev->gadget, USB_STATE_NOTATTACHED);
1324                 cdns3_hw_reset_eps_config(priv_dev);
1325         }
1326
1327         if (usb_ists & (USB_ISTS_L2ENTI | USB_ISTS_U3ENTI)) {
1328                 if (priv_dev->gadget_driver &&
1329                     priv_dev->gadget_driver->suspend) {
1330                         spin_unlock(&priv_dev->lock);
1331                         priv_dev->gadget_driver->suspend(&priv_dev->gadget);
1332                         spin_lock(&priv_dev->lock);
1333                 }
1334         }
1335
1336         if (usb_ists & (USB_ISTS_L2EXTI | USB_ISTS_U3EXTI)) {
1337                 if (priv_dev->gadget_driver &&
1338                     priv_dev->gadget_driver->resume) {
1339                         spin_unlock(&priv_dev->lock);
1340                         priv_dev->gadget_driver->resume(&priv_dev->gadget);
1341                         spin_lock(&priv_dev->lock);
1342                 }
1343         }
1344
1345         /* reset*/
1346         if (usb_ists & (USB_ISTS_UWRESI | USB_ISTS_UHRESI | USB_ISTS_U2RESI)) {
1347                 if (priv_dev->gadget_driver) {
1348                         spin_unlock(&priv_dev->lock);
1349                         usb_gadget_udc_reset(&priv_dev->gadget,
1350                                              priv_dev->gadget_driver);
1351                         spin_lock(&priv_dev->lock);
1352
1353                         /*read again to check the actual speed*/
1354                         speed = cdns3_get_speed(priv_dev);
1355                         priv_dev->gadget.speed = speed;
1356                         cdns3_hw_reset_eps_config(priv_dev);
1357                         cdns3_ep0_config(priv_dev);
1358                 }
1359         }
1360 }
1361
1362 /**
1363  * cdns3_device_irq_handler- interrupt handler for device part of controller
1364  *
1365  * @irq: irq number for cdns3 core device
1366  * @data: structure of cdns3
1367  *
1368  * Returns IRQ_HANDLED or IRQ_NONE
1369  */
1370 static irqreturn_t cdns3_device_irq_handler(int irq, void *data)
1371 {
1372         struct cdns3_device *priv_dev;
1373         struct cdns3 *cdns = data;
1374         irqreturn_t ret = IRQ_NONE;
1375         u32 reg;
1376
1377         priv_dev = cdns->gadget_dev;
1378
1379         /* check USB device interrupt */
1380         reg = readl(&priv_dev->regs->usb_ists);
1381         if (reg) {
1382                 /* After masking interrupts the new interrupts won't be
1383                  * reported in usb_ists/ep_ists. In order to not lose some
1384                  * of them driver disables only detected interrupts.
1385                  * They will be enabled ASAP after clearing source of
1386                  * interrupt. This an unusual behavior only applies to
1387                  * usb_ists register.
1388                  */
1389                 reg = ~reg & readl(&priv_dev->regs->usb_ien);
1390                 /* mask deferred interrupt. */
1391                 writel(reg, &priv_dev->regs->usb_ien);
1392                 ret = IRQ_WAKE_THREAD;
1393         }
1394
1395         /* check endpoint interrupt */
1396         reg = readl(&priv_dev->regs->ep_ists);
1397         if (reg) {
1398                 writel(0, &priv_dev->regs->ep_ien);
1399                 ret = IRQ_WAKE_THREAD;
1400         }
1401
1402         return ret;
1403 }
1404
1405 /**
1406  * cdns3_device_thread_irq_handler- interrupt handler for device part
1407  * of controller
1408  *
1409  * @irq: irq number for cdns3 core device
1410  * @data: structure of cdns3
1411  *
1412  * Returns IRQ_HANDLED or IRQ_NONE
1413  */
1414 static irqreturn_t cdns3_device_thread_irq_handler(int irq, void *data)
1415 {
1416         struct cdns3_device *priv_dev;
1417         struct cdns3 *cdns = data;
1418         irqreturn_t ret = IRQ_NONE;
1419         unsigned long flags;
1420         int bit;
1421         u32 reg;
1422
1423         priv_dev = cdns->gadget_dev;
1424         spin_lock_irqsave(&priv_dev->lock, flags);
1425
1426         reg = readl(&priv_dev->regs->usb_ists);
1427         if (reg) {
1428                 writel(reg, &priv_dev->regs->usb_ists);
1429                 writel(USB_IEN_INIT, &priv_dev->regs->usb_ien);
1430                 cdns3_check_usb_interrupt_proceed(priv_dev, reg);
1431                 ret = IRQ_HANDLED;
1432         }
1433
1434         reg = readl(&priv_dev->regs->ep_ists);
1435
1436         /* handle default endpoint OUT */
1437         if (reg & EP_ISTS_EP_OUT0) {
1438                 cdns3_check_ep0_interrupt_proceed(priv_dev, USB_DIR_OUT);
1439                 ret = IRQ_HANDLED;
1440         }
1441
1442         /* handle default endpoint IN */
1443         if (reg & EP_ISTS_EP_IN0) {
1444                 cdns3_check_ep0_interrupt_proceed(priv_dev, USB_DIR_IN);
1445                 ret = IRQ_HANDLED;
1446         }
1447
1448         /* check if interrupt from non default endpoint, if no exit */
1449         reg &= ~(EP_ISTS_EP_OUT0 | EP_ISTS_EP_IN0);
1450         if (!reg)
1451                 goto irqend;
1452
1453         for_each_set_bit(bit, (unsigned long *)&reg,
1454                          sizeof(u32) * BITS_PER_BYTE) {
1455                 cdns3_check_ep_interrupt_proceed(priv_dev->eps[bit]);
1456                 ret = IRQ_HANDLED;
1457         }
1458
1459 irqend:
1460         writel(~0, &priv_dev->regs->ep_ien);
1461         spin_unlock_irqrestore(&priv_dev->lock, flags);
1462
1463         return ret;
1464 }
1465
1466 /**
1467  * cdns3_ep_onchip_buffer_reserve - Try to reserve onchip buf for EP
1468  *
1469  * The real reservation will occur during write to EP_CFG register,
1470  * this function is used to check if the 'size' reservation is allowed.
1471  *
1472  * @priv_dev: extended gadget object
1473  * @size: the size (KB) for EP would like to allocate
1474  * @is_in: endpoint direction
1475  *
1476  * Return 0 if the required size can met or negative value on failure
1477  */
1478 static int cdns3_ep_onchip_buffer_reserve(struct cdns3_device *priv_dev,
1479                                           int size, int is_in)
1480 {
1481         int remained;
1482
1483         /* 2KB are reserved for EP0*/
1484         remained = priv_dev->onchip_buffers - priv_dev->onchip_used_size - 2;
1485
1486         if (is_in) {
1487                 if (remained < size)
1488                         return -EPERM;
1489
1490                 priv_dev->onchip_used_size += size;
1491         } else {
1492                 int required;
1493
1494                 /**
1495                  *  ALL OUT EPs are shared the same chunk onchip memory, so
1496                  * driver checks if it already has assigned enough buffers
1497                  */
1498                 if (priv_dev->out_mem_is_allocated >= size)
1499                         return 0;
1500
1501                 required = size - priv_dev->out_mem_is_allocated;
1502
1503                 if (required > remained)
1504                         return -EPERM;
1505
1506                 priv_dev->out_mem_is_allocated += required;
1507                 priv_dev->onchip_used_size += required;
1508         }
1509
1510         return 0;
1511 }
1512
1513 void cdns3_configure_dmult(struct cdns3_device *priv_dev,
1514                            struct cdns3_endpoint *priv_ep)
1515 {
1516         struct cdns3_usb_regs __iomem *regs = priv_dev->regs;
1517
1518         /* For dev_ver > DEV_VER_V2 DMULT is configured per endpoint */
1519         if (priv_dev->dev_ver <= DEV_VER_V2)
1520                 writel(USB_CONF_DMULT, &regs->usb_conf);
1521
1522         if (priv_dev->dev_ver == DEV_VER_V2)
1523                 writel(USB_CONF2_EN_TDL_TRB, &regs->usb_conf2);
1524
1525         if (priv_dev->dev_ver >= DEV_VER_V3 && priv_ep) {
1526                 u32 mask;
1527
1528                 if (priv_ep->dir)
1529                         mask = BIT(priv_ep->num + 16);
1530                 else
1531                         mask = BIT(priv_ep->num);
1532
1533                 if (priv_ep->type != USB_ENDPOINT_XFER_ISOC) {
1534                         cdns3_set_register_bit(&regs->tdl_from_trb, mask);
1535                         cdns3_set_register_bit(&regs->tdl_beh, mask);
1536                         cdns3_set_register_bit(&regs->tdl_beh2, mask);
1537                         cdns3_set_register_bit(&regs->dma_adv_td, mask);
1538                 }
1539
1540                 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && !priv_ep->dir)
1541                         cdns3_set_register_bit(&regs->tdl_from_trb, mask);
1542
1543                 cdns3_set_register_bit(&regs->dtrans, mask);
1544         }
1545 }
1546
1547 /**
1548  * cdns3_ep_config Configure hardware endpoint
1549  * @priv_ep: extended endpoint object
1550  */
1551 void cdns3_ep_config(struct cdns3_endpoint *priv_ep)
1552 {
1553         bool is_iso_ep = (priv_ep->type == USB_ENDPOINT_XFER_ISOC);
1554         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1555         u32 bEndpointAddress = priv_ep->num | priv_ep->dir;
1556         u32 max_packet_size = 0;
1557         u8 maxburst = 0;
1558         u32 ep_cfg = 0;
1559         u8 buffering;
1560         u8 mult = 0;
1561         int ret;
1562
1563         buffering = CDNS3_EP_BUF_SIZE - 1;
1564
1565         cdns3_configure_dmult(priv_dev, priv_ep);
1566
1567         switch (priv_ep->type) {
1568         case USB_ENDPOINT_XFER_INT:
1569                 ep_cfg = EP_CFG_EPTYPE(USB_ENDPOINT_XFER_INT);
1570
1571                 if ((priv_dev->dev_ver == DEV_VER_V2 && !priv_ep->dir) ||
1572                     priv_dev->dev_ver > DEV_VER_V2)
1573                         ep_cfg |= EP_CFG_TDL_CHK;
1574                 break;
1575         case USB_ENDPOINT_XFER_BULK:
1576                 ep_cfg = EP_CFG_EPTYPE(USB_ENDPOINT_XFER_BULK);
1577
1578                 if ((priv_dev->dev_ver == DEV_VER_V2  && !priv_ep->dir) ||
1579                     priv_dev->dev_ver > DEV_VER_V2)
1580                         ep_cfg |= EP_CFG_TDL_CHK;
1581                 break;
1582         default:
1583                 ep_cfg = EP_CFG_EPTYPE(USB_ENDPOINT_XFER_ISOC);
1584                 mult = CDNS3_EP_ISO_HS_MULT - 1;
1585                 buffering = mult + 1;
1586         }
1587
1588         switch (priv_dev->gadget.speed) {
1589         case USB_SPEED_FULL:
1590                 max_packet_size = is_iso_ep ? 1023 : 64;
1591                 break;
1592         case USB_SPEED_HIGH:
1593                 max_packet_size = is_iso_ep ? 1024 : 512;
1594                 break;
1595         case USB_SPEED_SUPER:
1596                 /* It's limitation that driver assumes in driver. */
1597                 mult = 0;
1598                 max_packet_size = 1024;
1599                 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC) {
1600                         maxburst = CDNS3_EP_ISO_SS_BURST - 1;
1601                         buffering = (mult + 1) *
1602                                     (maxburst + 1);
1603
1604                         if (priv_ep->interval > 1)
1605                                 buffering++;
1606                 } else {
1607                         maxburst = CDNS3_EP_BUF_SIZE - 1;
1608                 }
1609                 break;
1610         default:
1611                 /* all other speed are not supported */
1612                 return;
1613         }
1614
1615         if (max_packet_size == 1024)
1616                 priv_ep->trb_burst_size = 128;
1617         else if (max_packet_size >= 512)
1618                 priv_ep->trb_burst_size = 64;
1619         else
1620                 priv_ep->trb_burst_size = 16;
1621
1622         ret = cdns3_ep_onchip_buffer_reserve(priv_dev, buffering + 1,
1623                                              !!priv_ep->dir);
1624         if (ret) {
1625                 dev_err(priv_dev->dev, "onchip mem is full, ep is invalid\n");
1626                 return;
1627         }
1628
1629         ep_cfg |= EP_CFG_MAXPKTSIZE(max_packet_size) |
1630                   EP_CFG_MULT(mult) |
1631                   EP_CFG_BUFFERING(buffering) |
1632                   EP_CFG_MAXBURST(maxburst);
1633
1634         cdns3_select_ep(priv_dev, bEndpointAddress);
1635         writel(ep_cfg, &priv_dev->regs->ep_cfg);
1636
1637         dev_dbg(priv_dev->dev, "Configure %s: with val %08x\n",
1638                 priv_ep->name, ep_cfg);
1639 }
1640
1641 /* Find correct direction for HW endpoint according to description */
1642 static int cdns3_ep_dir_is_correct(struct usb_endpoint_descriptor *desc,
1643                                    struct cdns3_endpoint *priv_ep)
1644 {
1645         return (priv_ep->endpoint.caps.dir_in && usb_endpoint_dir_in(desc)) ||
1646                (priv_ep->endpoint.caps.dir_out && usb_endpoint_dir_out(desc));
1647 }
1648
1649 static struct
1650 cdns3_endpoint *cdns3_find_available_ep(struct cdns3_device *priv_dev,
1651                                         struct usb_endpoint_descriptor *desc)
1652 {
1653         struct usb_ep *ep;
1654         struct cdns3_endpoint *priv_ep;
1655
1656         list_for_each_entry(ep, &priv_dev->gadget.ep_list, ep_list) {
1657                 unsigned long num;
1658                 /* ep name pattern likes epXin or epXout */
1659                 char c[2] = {ep->name[2], '\0'};
1660
1661                 num = simple_strtoul(c, NULL, 10);
1662
1663                 priv_ep = ep_to_cdns3_ep(ep);
1664                 if (cdns3_ep_dir_is_correct(desc, priv_ep)) {
1665                         if (!(priv_ep->flags & EP_CLAIMED)) {
1666                                 priv_ep->num  = num;
1667                                 return priv_ep;
1668                         }
1669                 }
1670         }
1671
1672         return ERR_PTR(-ENOENT);
1673 }
1674
1675 /*
1676  *  Cadence IP has one limitation that all endpoints must be configured
1677  * (Type & MaxPacketSize) before setting configuration through hardware
1678  * register, it means we can't change endpoints configuration after
1679  * set_configuration.
1680  *
1681  * This function set EP_CLAIMED flag which is added when the gadget driver
1682  * uses usb_ep_autoconfig to configure specific endpoint;
1683  * When the udc driver receives set_configurion request,
1684  * it goes through all claimed endpoints, and configure all endpoints
1685  * accordingly.
1686  *
1687  * At usb_ep_ops.enable/disable, we only enable and disable endpoint through
1688  * ep_cfg register which can be changed after set_configuration, and do
1689  * some software operation accordingly.
1690  */
1691 static struct
1692 usb_ep *cdns3_gadget_match_ep(struct usb_gadget *gadget,
1693                               struct usb_endpoint_descriptor *desc,
1694                               struct usb_ss_ep_comp_descriptor *comp_desc)
1695 {
1696         struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
1697         struct cdns3_endpoint *priv_ep;
1698         unsigned long flags;
1699
1700         priv_ep = cdns3_find_available_ep(priv_dev, desc);
1701         if (IS_ERR(priv_ep)) {
1702                 dev_err(priv_dev->dev, "no available ep\n");
1703                 return NULL;
1704         }
1705
1706         dev_dbg(priv_dev->dev, "match endpoint: %s\n", priv_ep->name);
1707
1708         spin_lock_irqsave(&priv_dev->lock, flags);
1709         priv_ep->endpoint.desc = desc;
1710         priv_ep->dir  = usb_endpoint_dir_in(desc) ? USB_DIR_IN : USB_DIR_OUT;
1711         priv_ep->type = usb_endpoint_type(desc);
1712         priv_ep->flags |= EP_CLAIMED;
1713         priv_ep->interval = desc->bInterval ? BIT(desc->bInterval - 1) : 0;
1714
1715         spin_unlock_irqrestore(&priv_dev->lock, flags);
1716         return &priv_ep->endpoint;
1717 }
1718
1719 /**
1720  * cdns3_gadget_ep_alloc_request Allocates request
1721  * @ep: endpoint object associated with request
1722  * @gfp_flags: gfp flags
1723  *
1724  * Returns allocated request address, NULL on allocation error
1725  */
1726 struct usb_request *cdns3_gadget_ep_alloc_request(struct usb_ep *ep,
1727                                                   gfp_t gfp_flags)
1728 {
1729         struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep);
1730         struct cdns3_request *priv_req;
1731
1732         priv_req = kzalloc(sizeof(*priv_req), gfp_flags);
1733         if (!priv_req)
1734                 return NULL;
1735
1736         priv_req->priv_ep = priv_ep;
1737
1738         trace_cdns3_alloc_request(priv_req);
1739         return &priv_req->request;
1740 }
1741
1742 /**
1743  * cdns3_gadget_ep_free_request Free memory occupied by request
1744  * @ep: endpoint object associated with request
1745  * @request: request to free memory
1746  */
1747 void cdns3_gadget_ep_free_request(struct usb_ep *ep,
1748                                   struct usb_request *request)
1749 {
1750         struct cdns3_request *priv_req = to_cdns3_request(request);
1751
1752         if (priv_req->aligned_buf)
1753                 priv_req->aligned_buf->in_use = 0;
1754
1755         trace_cdns3_free_request(priv_req);
1756         kfree(priv_req);
1757 }
1758
1759 /**
1760  * cdns3_gadget_ep_enable Enable endpoint
1761  * @ep: endpoint object
1762  * @desc: endpoint descriptor
1763  *
1764  * Returns 0 on success, error code elsewhere
1765  */
1766 static int cdns3_gadget_ep_enable(struct usb_ep *ep,
1767                                   const struct usb_endpoint_descriptor *desc)
1768 {
1769         struct cdns3_endpoint *priv_ep;
1770         struct cdns3_device *priv_dev;
1771         u32 reg = EP_STS_EN_TRBERREN;
1772         u32 bEndpointAddress;
1773         unsigned long flags;
1774         int enable = 1;
1775         int ret;
1776         int val;
1777
1778         priv_ep = ep_to_cdns3_ep(ep);
1779         priv_dev = priv_ep->cdns3_dev;
1780
1781         if (!ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT) {
1782                 dev_dbg(priv_dev->dev, "usbss: invalid parameters\n");
1783                 return -EINVAL;
1784         }
1785
1786         if (!desc->wMaxPacketSize) {
1787                 dev_err(priv_dev->dev, "usbss: missing wMaxPacketSize\n");
1788                 return -EINVAL;
1789         }
1790
1791         if (WARN_ON(priv_ep->flags & EP_ENABLED))
1792                 return 0;
1793
1794         spin_lock_irqsave(&priv_dev->lock, flags);
1795
1796         priv_ep->endpoint.desc = desc;
1797         priv_ep->type = usb_endpoint_type(desc);
1798         priv_ep->interval = desc->bInterval ? BIT(desc->bInterval - 1) : 0;
1799
1800         if (priv_ep->interval > ISO_MAX_INTERVAL &&
1801             priv_ep->type == USB_ENDPOINT_XFER_ISOC) {
1802                 dev_err(priv_dev->dev, "Driver is limited to %d period\n",
1803                         ISO_MAX_INTERVAL);
1804
1805                 ret =  -EINVAL;
1806                 goto exit;
1807         }
1808
1809         ret = cdns3_allocate_trb_pool(priv_ep);
1810
1811         if (ret)
1812                 goto exit;
1813
1814         bEndpointAddress = priv_ep->num | priv_ep->dir;
1815         cdns3_select_ep(priv_dev, bEndpointAddress);
1816
1817         trace_cdns3_gadget_ep_enable(priv_ep);
1818
1819         writel(EP_CMD_EPRST, &priv_dev->regs->ep_cmd);
1820
1821         ret = readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
1822                                         !(val & (EP_CMD_CSTALL | EP_CMD_EPRST)),
1823                                         1000);
1824
1825         if (unlikely(ret)) {
1826                 cdns3_free_trb_pool(priv_ep);
1827                 ret =  -EINVAL;
1828                 goto exit;
1829         }
1830
1831         /* enable interrupt for selected endpoint */
1832         cdns3_set_register_bit(&priv_dev->regs->ep_ien,
1833                                BIT(cdns3_ep_addr_to_index(bEndpointAddress)));
1834
1835         if (priv_dev->dev_ver < DEV_VER_V2)
1836                 cdns3_wa2_enable_detection(priv_dev, priv_ep, reg);
1837
1838         writel(reg, &priv_dev->regs->ep_sts_en);
1839
1840         /*
1841          * For some versions of controller at some point during ISO OUT traffic
1842          * DMA reads Transfer Ring for the EP which has never got doorbell.
1843          * This issue was detected only on simulation, but to avoid this issue
1844          * driver add protection against it. To fix it driver enable ISO OUT
1845          * endpoint before setting DRBL. This special treatment of ISO OUT
1846          * endpoints are recommended by controller specification.
1847          */
1848         if (priv_ep->type == USB_ENDPOINT_XFER_ISOC  && !priv_ep->dir)
1849                 enable = 0;
1850
1851         if (enable)
1852                 cdns3_set_register_bit(&priv_dev->regs->ep_cfg, EP_CFG_ENABLE);
1853
1854         ep->desc = desc;
1855         priv_ep->flags &= ~(EP_PENDING_REQUEST | EP_STALLED | EP_STALL_PENDING |
1856                             EP_QUIRK_ISO_OUT_EN | EP_QUIRK_EXTRA_BUF_EN);
1857         priv_ep->flags |= EP_ENABLED | EP_UPDATE_EP_TRBADDR;
1858         priv_ep->wa1_set = 0;
1859         priv_ep->enqueue = 0;
1860         priv_ep->dequeue = 0;
1861         reg = readl(&priv_dev->regs->ep_sts);
1862         priv_ep->pcs = !!EP_STS_CCS(reg);
1863         priv_ep->ccs = !!EP_STS_CCS(reg);
1864         /* one TRB is reserved for link TRB used in DMULT mode*/
1865         priv_ep->free_trbs = priv_ep->num_trbs - 1;
1866 exit:
1867         spin_unlock_irqrestore(&priv_dev->lock, flags);
1868
1869         return ret;
1870 }
1871
1872 /**
1873  * cdns3_gadget_ep_disable Disable endpoint
1874  * @ep: endpoint object
1875  *
1876  * Returns 0 on success, error code elsewhere
1877  */
1878 static int cdns3_gadget_ep_disable(struct usb_ep *ep)
1879 {
1880         struct cdns3_endpoint *priv_ep;
1881         struct cdns3_request *priv_req;
1882         struct cdns3_device *priv_dev;
1883         struct usb_request *request;
1884         unsigned long flags;
1885         int ret = 0;
1886         u32 ep_cfg;
1887         int val;
1888
1889         if (!ep) {
1890                 pr_err("usbss: invalid parameters\n");
1891                 return -EINVAL;
1892         }
1893
1894         priv_ep = ep_to_cdns3_ep(ep);
1895         priv_dev = priv_ep->cdns3_dev;
1896
1897         if (WARN_ON(!(priv_ep->flags & EP_ENABLED)))
1898                 return 0;
1899
1900         spin_lock_irqsave(&priv_dev->lock, flags);
1901
1902         trace_cdns3_gadget_ep_disable(priv_ep);
1903
1904         cdns3_select_ep(priv_dev, ep->desc->bEndpointAddress);
1905
1906         ep_cfg = readl(&priv_dev->regs->ep_cfg);
1907         ep_cfg &= ~EP_CFG_ENABLE;
1908         writel(ep_cfg, &priv_dev->regs->ep_cfg);
1909
1910         /**
1911          * Driver needs some time before resetting endpoint.
1912          * It need waits for clearing DBUSY bit or for timeout expired.
1913          * 10us is enough time for controller to stop transfer.
1914          */
1915         readl_poll_timeout_atomic(&priv_dev->regs->ep_sts, val,
1916                                   !(val & EP_STS_DBUSY), 10);
1917         writel(EP_CMD_EPRST, &priv_dev->regs->ep_cmd);
1918
1919         readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
1920                                   !(val & (EP_CMD_CSTALL | EP_CMD_EPRST)),
1921                                   1000);
1922         if (unlikely(ret))
1923                 dev_err(priv_dev->dev, "Timeout: %s resetting failed.\n",
1924                         priv_ep->name);
1925
1926         while (!list_empty(&priv_ep->pending_req_list)) {
1927                 request = cdns3_next_request(&priv_ep->pending_req_list);
1928
1929                 cdns3_gadget_giveback(priv_ep, to_cdns3_request(request),
1930                                       -ESHUTDOWN);
1931         }
1932
1933         while (!list_empty(&priv_ep->wa2_descmiss_req_list)) {
1934                 priv_req = cdns3_next_priv_request(&priv_ep->wa2_descmiss_req_list);
1935
1936                 kfree(priv_req->request.buf);
1937                 cdns3_gadget_ep_free_request(&priv_ep->endpoint,
1938                                              &priv_req->request);
1939                 list_del_init(&priv_req->list);
1940                 --priv_ep->wa2_counter;
1941         }
1942
1943         while (!list_empty(&priv_ep->deferred_req_list)) {
1944                 request = cdns3_next_request(&priv_ep->deferred_req_list);
1945
1946                 cdns3_gadget_giveback(priv_ep, to_cdns3_request(request),
1947                                       -ESHUTDOWN);
1948         }
1949
1950         priv_ep->descmis_req = NULL;
1951
1952         ep->desc = NULL;
1953         priv_ep->flags &= ~EP_ENABLED;
1954
1955         spin_unlock_irqrestore(&priv_dev->lock, flags);
1956
1957         return ret;
1958 }
1959
1960 /**
1961  * cdns3_gadget_ep_queue Transfer data on endpoint
1962  * @ep: endpoint object
1963  * @request: request object
1964  * @gfp_flags: gfp flags
1965  *
1966  * Returns 0 on success, error code elsewhere
1967  */
1968 static int __cdns3_gadget_ep_queue(struct usb_ep *ep,
1969                                    struct usb_request *request,
1970                                    gfp_t gfp_flags)
1971 {
1972         struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep);
1973         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1974         struct cdns3_request *priv_req;
1975         int ret = 0;
1976
1977         request->actual = 0;
1978         request->status = -EINPROGRESS;
1979         priv_req = to_cdns3_request(request);
1980         trace_cdns3_ep_queue(priv_req);
1981
1982         if (priv_dev->dev_ver < DEV_VER_V2) {
1983                 ret = cdns3_wa2_gadget_ep_queue(priv_dev, priv_ep,
1984                                                 priv_req);
1985
1986                 if (ret == EINPROGRESS)
1987                         return 0;
1988         }
1989
1990         ret = cdns3_prepare_aligned_request_buf(priv_req);
1991         if (ret < 0)
1992                 return ret;
1993
1994         ret = usb_gadget_map_request(&priv_dev->gadget, request,
1995                                      usb_endpoint_dir_in(ep->desc));
1996         if (ret)
1997                 return ret;
1998
1999         list_add_tail(&request->list, &priv_ep->deferred_req_list);
2000
2001         /*
2002          * If hardware endpoint configuration has not been set yet then
2003          * just queue request in deferred list. Transfer will be started in
2004          * cdns3_set_hw_configuration.
2005          */
2006         if (priv_dev->hw_configured_flag && !(priv_ep->flags & EP_STALLED) &&
2007             !(priv_ep->flags & EP_STALL_PENDING))
2008                 cdns3_start_all_request(priv_dev, priv_ep);
2009
2010         return 0;
2011 }
2012
2013 static int cdns3_gadget_ep_queue(struct usb_ep *ep, struct usb_request *request,
2014                                  gfp_t gfp_flags)
2015 {
2016         struct usb_request *zlp_request;
2017         struct cdns3_endpoint *priv_ep;
2018         struct cdns3_device *priv_dev;
2019         unsigned long flags;
2020         int ret;
2021
2022         if (!request || !ep)
2023                 return -EINVAL;
2024
2025         priv_ep = ep_to_cdns3_ep(ep);
2026         priv_dev = priv_ep->cdns3_dev;
2027
2028         spin_lock_irqsave(&priv_dev->lock, flags);
2029
2030         ret = __cdns3_gadget_ep_queue(ep, request, gfp_flags);
2031
2032         if (ret == 0 && request->zero && request->length &&
2033             (request->length % ep->maxpacket == 0)) {
2034                 struct cdns3_request *priv_req;
2035
2036                 zlp_request = cdns3_gadget_ep_alloc_request(ep, GFP_ATOMIC);
2037                 zlp_request->buf = priv_dev->zlp_buf;
2038                 zlp_request->length = 0;
2039
2040                 priv_req = to_cdns3_request(zlp_request);
2041                 priv_req->flags |= REQUEST_ZLP;
2042
2043                 dev_dbg(priv_dev->dev, "Queuing ZLP for endpoint: %s\n",
2044                         priv_ep->name);
2045                 ret = __cdns3_gadget_ep_queue(ep, zlp_request, gfp_flags);
2046         }
2047
2048         spin_unlock_irqrestore(&priv_dev->lock, flags);
2049         return ret;
2050 }
2051
2052 /**
2053  * cdns3_gadget_ep_dequeue Remove request from transfer queue
2054  * @ep: endpoint object associated with request
2055  * @request: request object
2056  *
2057  * Returns 0 on success, error code elsewhere
2058  */
2059 int cdns3_gadget_ep_dequeue(struct usb_ep *ep,
2060                             struct usb_request *request)
2061 {
2062         struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep);
2063         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
2064         struct usb_request *req, *req_temp;
2065         struct cdns3_request *priv_req;
2066         struct cdns3_trb *link_trb;
2067         unsigned long flags;
2068         int ret = 0;
2069
2070         if (!ep || !request || !ep->desc)
2071                 return -EINVAL;
2072
2073         spin_lock_irqsave(&priv_dev->lock, flags);
2074
2075         priv_req = to_cdns3_request(request);
2076
2077         trace_cdns3_ep_dequeue(priv_req);
2078
2079         cdns3_select_ep(priv_dev, ep->desc->bEndpointAddress);
2080
2081         list_for_each_entry_safe(req, req_temp, &priv_ep->pending_req_list,
2082                                  list) {
2083                 if (request == req)
2084                         goto found;
2085         }
2086
2087         list_for_each_entry_safe(req, req_temp, &priv_ep->deferred_req_list,
2088                                  list) {
2089                 if (request == req)
2090                         goto found;
2091         }
2092
2093         goto not_found;
2094
2095 found:
2096
2097         if (priv_ep->wa1_trb == priv_req->trb)
2098                 cdns3_wa1_restore_cycle_bit(priv_ep);
2099
2100         link_trb = priv_req->trb;
2101         cdns3_move_deq_to_next_trb(priv_req);
2102         cdns3_gadget_giveback(priv_ep, priv_req, -ECONNRESET);
2103
2104         /* Update ring */
2105         request = cdns3_next_request(&priv_ep->deferred_req_list);
2106         if (request) {
2107                 priv_req = to_cdns3_request(request);
2108
2109                 link_trb->buffer = TRB_BUFFER(priv_ep->trb_pool_dma +
2110                                               (priv_req->start_trb * TRB_SIZE));
2111                 link_trb->control = (link_trb->control & TRB_CYCLE) |
2112                                     TRB_TYPE(TRB_LINK) | TRB_CHAIN | TRB_TOGGLE;
2113         } else {
2114                 priv_ep->flags |= EP_UPDATE_EP_TRBADDR;
2115         }
2116
2117 not_found:
2118         spin_unlock_irqrestore(&priv_dev->lock, flags);
2119         return ret;
2120 }
2121
2122 /**
2123  * __cdns3_gadget_ep_set_halt Sets stall on selected endpoint
2124  * Should be called after acquiring spin_lock and selecting ep
2125  * @ep: endpoint object to set stall on.
2126  */
2127 void __cdns3_gadget_ep_set_halt(struct cdns3_endpoint *priv_ep)
2128 {
2129         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
2130
2131         trace_cdns3_halt(priv_ep, 1, 0);
2132
2133         if (!(priv_ep->flags & EP_STALLED)) {
2134                 u32 ep_sts_reg = readl(&priv_dev->regs->ep_sts);
2135
2136                 if (!(ep_sts_reg & EP_STS_DBUSY))
2137                         cdns3_ep_stall_flush(priv_ep);
2138                 else
2139                         priv_ep->flags |= EP_STALL_PENDING;
2140         }
2141 }
2142
2143 /**
2144  * __cdns3_gadget_ep_clear_halt Clears stall on selected endpoint
2145  * Should be called after acquiring spin_lock and selecting ep
2146  * @ep: endpoint object to clear stall on
2147  */
2148 int __cdns3_gadget_ep_clear_halt(struct cdns3_endpoint *priv_ep)
2149 {
2150         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
2151         struct usb_request *request;
2152         int ret = 0;
2153         int val;
2154
2155         trace_cdns3_halt(priv_ep, 0, 0);
2156
2157         writel(EP_CMD_CSTALL | EP_CMD_EPRST, &priv_dev->regs->ep_cmd);
2158
2159         /* wait for EPRST cleared */
2160         readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
2161                                   !(val & EP_CMD_EPRST), 100);
2162         if (ret)
2163                 return -EINVAL;
2164
2165         priv_ep->flags &= ~(EP_STALLED | EP_STALL_PENDING);
2166
2167         request = cdns3_next_request(&priv_ep->pending_req_list);
2168
2169         if (request)
2170                 cdns3_rearm_transfer(priv_ep, 1);
2171
2172         cdns3_start_all_request(priv_dev, priv_ep);
2173         return ret;
2174 }
2175
2176 /**
2177  * cdns3_gadget_ep_set_halt Sets/clears stall on selected endpoint
2178  * @ep: endpoint object to set/clear stall on
2179  * @value: 1 for set stall, 0 for clear stall
2180  *
2181  * Returns 0 on success, error code elsewhere
2182  */
2183 int cdns3_gadget_ep_set_halt(struct usb_ep *ep, int value)
2184 {
2185         struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep);
2186         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
2187         unsigned long flags;
2188         int ret = 0;
2189
2190         if (!(priv_ep->flags & EP_ENABLED))
2191                 return -EPERM;
2192
2193         spin_lock_irqsave(&priv_dev->lock, flags);
2194
2195         cdns3_select_ep(priv_dev, ep->desc->bEndpointAddress);
2196
2197         if (!value) {
2198                 priv_ep->flags &= ~EP_WEDGE;
2199                 ret = __cdns3_gadget_ep_clear_halt(priv_ep);
2200         } else {
2201                 __cdns3_gadget_ep_set_halt(priv_ep);
2202         }
2203
2204         spin_unlock_irqrestore(&priv_dev->lock, flags);
2205
2206         return ret;
2207 }
2208
2209 extern const struct usb_ep_ops cdns3_gadget_ep0_ops;
2210
2211 static const struct usb_ep_ops cdns3_gadget_ep_ops = {
2212         .enable = cdns3_gadget_ep_enable,
2213         .disable = cdns3_gadget_ep_disable,
2214         .alloc_request = cdns3_gadget_ep_alloc_request,
2215         .free_request = cdns3_gadget_ep_free_request,
2216         .queue = cdns3_gadget_ep_queue,
2217         .dequeue = cdns3_gadget_ep_dequeue,
2218         .set_halt = cdns3_gadget_ep_set_halt,
2219         .set_wedge = cdns3_gadget_ep_set_wedge,
2220 };
2221
2222 /**
2223  * cdns3_gadget_get_frame Returns number of actual ITP frame
2224  * @gadget: gadget object
2225  *
2226  * Returns number of actual ITP frame
2227  */
2228 static int cdns3_gadget_get_frame(struct usb_gadget *gadget)
2229 {
2230         struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2231
2232         return readl(&priv_dev->regs->usb_itpn);
2233 }
2234
2235 int __cdns3_gadget_wakeup(struct cdns3_device *priv_dev)
2236 {
2237         enum usb_device_speed speed;
2238
2239         speed = cdns3_get_speed(priv_dev);
2240
2241         if (speed >= USB_SPEED_SUPER)
2242                 return 0;
2243
2244         /* Start driving resume signaling to indicate remote wakeup. */
2245         writel(USB_CONF_LGO_L0, &priv_dev->regs->usb_conf);
2246
2247         return 0;
2248 }
2249
2250 static int cdns3_gadget_wakeup(struct usb_gadget *gadget)
2251 {
2252         struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2253         unsigned long flags;
2254         int ret = 0;
2255
2256         spin_lock_irqsave(&priv_dev->lock, flags);
2257         ret = __cdns3_gadget_wakeup(priv_dev);
2258         spin_unlock_irqrestore(&priv_dev->lock, flags);
2259         return ret;
2260 }
2261
2262 static int cdns3_gadget_set_selfpowered(struct usb_gadget *gadget,
2263                                         int is_selfpowered)
2264 {
2265         struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2266         unsigned long flags;
2267
2268         spin_lock_irqsave(&priv_dev->lock, flags);
2269         priv_dev->is_selfpowered = !!is_selfpowered;
2270         spin_unlock_irqrestore(&priv_dev->lock, flags);
2271         return 0;
2272 }
2273
2274 static int cdns3_gadget_pullup(struct usb_gadget *gadget, int is_on)
2275 {
2276         struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2277
2278         if (is_on)
2279                 writel(USB_CONF_DEVEN, &priv_dev->regs->usb_conf);
2280         else
2281                 writel(USB_CONF_DEVDS, &priv_dev->regs->usb_conf);
2282
2283         return 0;
2284 }
2285
2286 static void cdns3_gadget_config(struct cdns3_device *priv_dev)
2287 {
2288         struct cdns3_usb_regs __iomem *regs = priv_dev->regs;
2289         u32 reg;
2290
2291         cdns3_ep0_config(priv_dev);
2292
2293         /* enable interrupts for endpoint 0 (in and out) */
2294         writel(EP_IEN_EP_OUT0 | EP_IEN_EP_IN0, &regs->ep_ien);
2295
2296         /*
2297          * Driver needs to modify LFPS minimal U1 Exit time for DEV_VER_TI_V1
2298          * revision of controller.
2299          */
2300         if (priv_dev->dev_ver == DEV_VER_TI_V1) {
2301                 reg = readl(&regs->dbg_link1);
2302
2303                 reg &= ~DBG_LINK1_LFPS_MIN_GEN_U1_EXIT_MASK;
2304                 reg |= DBG_LINK1_LFPS_MIN_GEN_U1_EXIT(0x55) |
2305                        DBG_LINK1_LFPS_MIN_GEN_U1_EXIT_SET;
2306                 writel(reg, &regs->dbg_link1);
2307         }
2308
2309         /*
2310          * By default some platforms has set protected access to memory.
2311          * This cause problem with cache, so driver restore non-secure
2312          * access to memory.
2313          */
2314         reg = readl(&regs->dma_axi_ctrl);
2315         reg |= DMA_AXI_CTRL_MARPROT(DMA_AXI_CTRL_NON_SECURE) |
2316                DMA_AXI_CTRL_MAWPROT(DMA_AXI_CTRL_NON_SECURE);
2317         writel(reg, &regs->dma_axi_ctrl);
2318
2319         /* enable generic interrupt*/
2320         writel(USB_IEN_INIT, &regs->usb_ien);
2321         writel(USB_CONF_CLK2OFFDS | USB_CONF_L1DS, &regs->usb_conf);
2322
2323         cdns3_configure_dmult(priv_dev, NULL);
2324
2325         cdns3_gadget_pullup(&priv_dev->gadget, 1);
2326 }
2327
2328 /**
2329  * cdns3_gadget_udc_start Gadget start
2330  * @gadget: gadget object
2331  * @driver: driver which operates on this gadget
2332  *
2333  * Returns 0 on success, error code elsewhere
2334  */
2335 static int cdns3_gadget_udc_start(struct usb_gadget *gadget,
2336                                   struct usb_gadget_driver *driver)
2337 {
2338         struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2339         unsigned long flags;
2340
2341         spin_lock_irqsave(&priv_dev->lock, flags);
2342         priv_dev->gadget_driver = driver;
2343         cdns3_gadget_config(priv_dev);
2344         spin_unlock_irqrestore(&priv_dev->lock, flags);
2345         return 0;
2346 }
2347
2348 /**
2349  * cdns3_gadget_udc_stop Stops gadget
2350  * @gadget: gadget object
2351  *
2352  * Returns 0
2353  */
2354 static int cdns3_gadget_udc_stop(struct usb_gadget *gadget)
2355 {
2356         struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2357         struct cdns3_endpoint *priv_ep;
2358         u32 bEndpointAddress;
2359         struct usb_ep *ep;
2360         int ret = 0;
2361         int val;
2362
2363         priv_dev->gadget_driver = NULL;
2364
2365         priv_dev->onchip_used_size = 0;
2366         priv_dev->out_mem_is_allocated = 0;
2367         priv_dev->gadget.speed = USB_SPEED_UNKNOWN;
2368
2369         list_for_each_entry(ep, &priv_dev->gadget.ep_list, ep_list) {
2370                 priv_ep = ep_to_cdns3_ep(ep);
2371                 bEndpointAddress = priv_ep->num | priv_ep->dir;
2372                 cdns3_select_ep(priv_dev, bEndpointAddress);
2373                 writel(EP_CMD_EPRST, &priv_dev->regs->ep_cmd);
2374                 readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
2375                                           !(val & EP_CMD_EPRST), 100);
2376         }
2377
2378         /* disable interrupt for device */
2379         writel(0, &priv_dev->regs->usb_ien);
2380         writel(USB_CONF_DEVDS, &priv_dev->regs->usb_conf);
2381
2382         return ret;
2383 }
2384
2385 static void cdns3_gadget_udc_set_speed(struct usb_gadget *gadget,
2386                                        enum usb_device_speed speed)
2387 {
2388         struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2389
2390         switch (speed) {
2391         case USB_SPEED_FULL:
2392                 writel(USB_CONF_SFORCE_FS, &priv_dev->regs->usb_conf);
2393                 writel(USB_CONF_USB3DIS, &priv_dev->regs->usb_conf);
2394                 break;
2395         case USB_SPEED_HIGH:
2396                 writel(USB_CONF_USB3DIS, &priv_dev->regs->usb_conf);
2397                 break;
2398         case USB_SPEED_SUPER:
2399                 break;
2400         default:
2401                 dev_err(cdns->dev, "invalid speed parameter %d\n",
2402                         speed);
2403         }
2404
2405         priv_dev->gadget.speed = speed;
2406 }
2407
2408 static const struct usb_gadget_ops cdns3_gadget_ops = {
2409         .get_frame = cdns3_gadget_get_frame,
2410         .wakeup = cdns3_gadget_wakeup,
2411         .set_selfpowered = cdns3_gadget_set_selfpowered,
2412         .pullup = cdns3_gadget_pullup,
2413         .udc_start = cdns3_gadget_udc_start,
2414         .udc_stop = cdns3_gadget_udc_stop,
2415         .match_ep = cdns3_gadget_match_ep,
2416         .udc_set_speed = cdns3_gadget_udc_set_speed,
2417 };
2418
2419 static void cdns3_free_all_eps(struct cdns3_device *priv_dev)
2420 {
2421         int i;
2422
2423         /* ep0 OUT point to ep0 IN. */
2424         priv_dev->eps[16] = NULL;
2425
2426         for (i = 0; i < CDNS3_ENDPOINTS_MAX_COUNT; i++)
2427                 if (priv_dev->eps[i]) {
2428                         cdns3_free_trb_pool(priv_dev->eps[i]);
2429                         devm_kfree(priv_dev->dev, priv_dev->eps[i]);
2430                 }
2431 }
2432
2433 /**
2434  * cdns3_init_eps Initializes software endpoints of gadget
2435  * @cdns3: extended gadget object
2436  *
2437  * Returns 0 on success, error code elsewhere
2438  */
2439 static int cdns3_init_eps(struct cdns3_device *priv_dev)
2440 {
2441         u32 ep_enabled_reg, iso_ep_reg;
2442         struct cdns3_endpoint *priv_ep;
2443         int ep_dir, ep_number;
2444         u32 ep_mask;
2445         int ret = 0;
2446         int i;
2447
2448         /* Read it from USB_CAP3 to USB_CAP5 */
2449         ep_enabled_reg = readl(&priv_dev->regs->usb_cap3);
2450         iso_ep_reg = readl(&priv_dev->regs->usb_cap4);
2451
2452         dev_dbg(priv_dev->dev, "Initializing non-zero endpoints\n");
2453
2454         for (i = 0; i < CDNS3_ENDPOINTS_MAX_COUNT; i++) {
2455                 ep_dir = i >> 4;        /* i div 16 */
2456                 ep_number = i & 0xF;    /* i % 16 */
2457                 ep_mask = BIT(i);
2458
2459                 if (!(ep_enabled_reg & ep_mask))
2460                         continue;
2461
2462                 if (ep_dir && !ep_number) {
2463                         priv_dev->eps[i] = priv_dev->eps[0];
2464                         continue;
2465                 }
2466
2467                 priv_ep = devm_kzalloc(priv_dev->dev, sizeof(*priv_ep),
2468                                        GFP_KERNEL);
2469                 if (!priv_ep) {
2470                         ret = -ENOMEM;
2471                         goto err;
2472                 }
2473
2474                 /* set parent of endpoint object */
2475                 priv_ep->cdns3_dev = priv_dev;
2476                 priv_dev->eps[i] = priv_ep;
2477                 priv_ep->num = ep_number;
2478                 priv_ep->dir = ep_dir ? USB_DIR_IN : USB_DIR_OUT;
2479
2480                 if (!ep_number) {
2481                         ret = cdns3_init_ep0(priv_dev, priv_ep);
2482                         if (ret) {
2483                                 dev_err(priv_dev->dev, "Failed to init ep0\n");
2484                                 goto err;
2485                         }
2486                 } else {
2487                         snprintf(priv_ep->name, sizeof(priv_ep->name), "ep%d%s",
2488                                  ep_number, !!ep_dir ? "in" : "out");
2489                         priv_ep->endpoint.name = priv_ep->name;
2490
2491                         usb_ep_set_maxpacket_limit(&priv_ep->endpoint,
2492                                                    CDNS3_EP_MAX_PACKET_LIMIT);
2493                         priv_ep->endpoint.max_streams = CDNS3_EP_MAX_STREAMS;
2494                         priv_ep->endpoint.ops = &cdns3_gadget_ep_ops;
2495                         if (ep_dir)
2496                                 priv_ep->endpoint.caps.dir_in = 1;
2497                         else
2498                                 priv_ep->endpoint.caps.dir_out = 1;
2499
2500                         if (iso_ep_reg & ep_mask)
2501                                 priv_ep->endpoint.caps.type_iso = 1;
2502
2503                         priv_ep->endpoint.caps.type_bulk = 1;
2504                         priv_ep->endpoint.caps.type_int = 1;
2505
2506                         list_add_tail(&priv_ep->endpoint.ep_list,
2507                                       &priv_dev->gadget.ep_list);
2508                 }
2509
2510                 priv_ep->flags = 0;
2511
2512                 dev_info(priv_dev->dev, "Initialized  %s support: %s %s\n",
2513                          priv_ep->name,
2514                          priv_ep->endpoint.caps.type_bulk ? "BULK, INT" : "",
2515                          priv_ep->endpoint.caps.type_iso ? "ISO" : "");
2516
2517                 INIT_LIST_HEAD(&priv_ep->pending_req_list);
2518                 INIT_LIST_HEAD(&priv_ep->deferred_req_list);
2519                 INIT_LIST_HEAD(&priv_ep->wa2_descmiss_req_list);
2520         }
2521
2522         return 0;
2523 err:
2524         cdns3_free_all_eps(priv_dev);
2525         return -ENOMEM;
2526 }
2527
2528 void cdns3_gadget_exit(struct cdns3 *cdns)
2529 {
2530         struct cdns3_device *priv_dev;
2531
2532         priv_dev = cdns->gadget_dev;
2533
2534         usb_del_gadget_udc(&priv_dev->gadget);
2535
2536         cdns3_free_all_eps(priv_dev);
2537
2538         while (!list_empty(&priv_dev->aligned_buf_list)) {
2539                 struct cdns3_aligned_buf *buf;
2540
2541                 buf = cdns3_next_align_buf(&priv_dev->aligned_buf_list);
2542                 dma_free_coherent(buf->buf);
2543
2544                 list_del(&buf->list);
2545                 kfree(buf);
2546         }
2547
2548         dma_free_coherent(priv_dev->setup_buf);
2549
2550         kfree(priv_dev->zlp_buf);
2551         kfree(priv_dev);
2552         cdns->gadget_dev = NULL;
2553         cdns3_drd_switch_gadget(cdns, 0);
2554 }
2555
2556 static int cdns3_gadget_start(struct cdns3 *cdns)
2557 {
2558         struct cdns3_device *priv_dev;
2559         u32 max_speed;
2560         int ret;
2561
2562         priv_dev = kzalloc(sizeof(*priv_dev), GFP_KERNEL);
2563         if (!priv_dev)
2564                 return -ENOMEM;
2565
2566         cdns->gadget_dev = priv_dev;
2567         priv_dev->sysdev = cdns->dev;
2568         priv_dev->dev = cdns->dev;
2569         priv_dev->regs = cdns->dev_regs;
2570
2571         dev_read_u32(priv_dev->dev, "cdns,on-chip-buff-size",
2572                      &priv_dev->onchip_buffers);
2573
2574         if (priv_dev->onchip_buffers <=  0) {
2575                 u32 reg = readl(&priv_dev->regs->usb_cap2);
2576
2577                 priv_dev->onchip_buffers = USB_CAP2_ACTUAL_MEM_SIZE(reg);
2578         }
2579
2580         if (!priv_dev->onchip_buffers)
2581                 priv_dev->onchip_buffers = 256;
2582
2583         max_speed = usb_get_maximum_speed(dev_ofnode(cdns->dev));
2584
2585         /* Check the maximum_speed parameter */
2586         switch (max_speed) {
2587         case USB_SPEED_FULL:
2588                 /* fall through */
2589         case USB_SPEED_HIGH:
2590                 /* fall through */
2591         case USB_SPEED_SUPER:
2592                 break;
2593         default:
2594                 dev_err(cdns->dev, "invalid maximum_speed parameter %d\n",
2595                         max_speed);
2596                 /* fall through */
2597         case USB_SPEED_UNKNOWN:
2598                 /* default to superspeed */
2599                 max_speed = USB_SPEED_SUPER;
2600                 break;
2601         }
2602
2603         /* fill gadget fields */
2604         priv_dev->gadget.max_speed = max_speed;
2605         priv_dev->gadget.speed = USB_SPEED_UNKNOWN;
2606         priv_dev->gadget.ops = &cdns3_gadget_ops;
2607         priv_dev->gadget.name = "cdns3-gadget";
2608 #ifndef __UBOOT__
2609         priv_dev->gadget.name = "usb-ss-gadget";
2610         priv_dev->gadget.sg_supported = 1;
2611         priv_dev->gadget.quirk_avoids_skb_reserve = 1;
2612 #endif
2613
2614         spin_lock_init(&priv_dev->lock);
2615         INIT_WORK(&priv_dev->pending_status_wq,
2616                   cdns3_pending_setup_status_handler);
2617
2618         /* initialize endpoint container */
2619         INIT_LIST_HEAD(&priv_dev->gadget.ep_list);
2620         INIT_LIST_HEAD(&priv_dev->aligned_buf_list);
2621
2622         ret = cdns3_init_eps(priv_dev);
2623         if (ret) {
2624                 dev_err(priv_dev->dev, "Failed to create endpoints\n");
2625                 goto err1;
2626         }
2627
2628         /* allocate memory for setup packet buffer */
2629         priv_dev->setup_buf =
2630                 dma_alloc_coherent(8, (unsigned long *)&priv_dev->setup_dma);
2631         if (!priv_dev->setup_buf) {
2632                 ret = -ENOMEM;
2633                 goto err2;
2634         }
2635
2636         priv_dev->dev_ver = readl(&priv_dev->regs->usb_cap6);
2637
2638         dev_dbg(priv_dev->dev, "Device Controller version: %08x\n",
2639                 readl(&priv_dev->regs->usb_cap6));
2640         dev_dbg(priv_dev->dev, "USB Capabilities:: %08x\n",
2641                 readl(&priv_dev->regs->usb_cap1));
2642         dev_dbg(priv_dev->dev, "On-Chip memory cnfiguration: %08x\n",
2643                 readl(&priv_dev->regs->usb_cap2));
2644
2645         priv_dev->dev_ver = GET_DEV_BASE_VERSION(priv_dev->dev_ver);
2646
2647         priv_dev->zlp_buf = kzalloc(CDNS3_EP_ZLP_BUF_SIZE, GFP_KERNEL);
2648         if (!priv_dev->zlp_buf) {
2649                 ret = -ENOMEM;
2650                 goto err3;
2651         }
2652
2653         /* add USB gadget device */
2654         ret = usb_add_gadget_udc((struct device *)priv_dev->dev,
2655                                  &priv_dev->gadget);
2656         if (ret < 0) {
2657                 dev_err(priv_dev->dev,
2658                         "Failed to register USB device controller\n");
2659                 goto err4;
2660         }
2661
2662         return 0;
2663 err4:
2664         kfree(priv_dev->zlp_buf);
2665 err3:
2666         dma_free_coherent(priv_dev->setup_buf);
2667 err2:
2668         cdns3_free_all_eps(priv_dev);
2669 err1:
2670         cdns->gadget_dev = NULL;
2671         return ret;
2672 }
2673
2674 static int __cdns3_gadget_init(struct cdns3 *cdns)
2675 {
2676         int ret = 0;
2677
2678         cdns3_drd_switch_gadget(cdns, 1);
2679
2680         ret = cdns3_gadget_start(cdns);
2681         if (ret)
2682                 return ret;
2683
2684         return 0;
2685 }
2686
2687 static int cdns3_gadget_suspend(struct cdns3 *cdns, bool do_wakeup)
2688 {
2689         struct cdns3_device *priv_dev = cdns->gadget_dev;
2690
2691         cdns3_disconnect_gadget(priv_dev);
2692
2693         priv_dev->gadget.speed = USB_SPEED_UNKNOWN;
2694         usb_gadget_set_state(&priv_dev->gadget, USB_STATE_NOTATTACHED);
2695         cdns3_hw_reset_eps_config(priv_dev);
2696
2697         /* disable interrupt for device */
2698         writel(0, &priv_dev->regs->usb_ien);
2699
2700         cdns3_gadget_pullup(&priv_dev->gadget, 0);
2701
2702         return 0;
2703 }
2704
2705 static int cdns3_gadget_resume(struct cdns3 *cdns, bool hibernated)
2706 {
2707         struct cdns3_device *priv_dev = cdns->gadget_dev;
2708
2709         if (!priv_dev->gadget_driver)
2710                 return 0;
2711
2712         cdns3_gadget_config(priv_dev);
2713
2714         return 0;
2715 }
2716
2717 /**
2718  * cdns3_gadget_init - initialize device structure
2719  *
2720  * cdns: cdns3 instance
2721  *
2722  * This function initializes the gadget.
2723  */
2724 int cdns3_gadget_init(struct cdns3 *cdns)
2725 {
2726         struct cdns3_role_driver *rdrv;
2727
2728         rdrv = devm_kzalloc(cdns->dev, sizeof(*rdrv), GFP_KERNEL);
2729         if (!rdrv)
2730                 return -ENOMEM;
2731
2732         rdrv->start     = __cdns3_gadget_init;
2733         rdrv->stop      = cdns3_gadget_exit;
2734         rdrv->suspend   = cdns3_gadget_suspend;
2735         rdrv->resume    = cdns3_gadget_resume;
2736         rdrv->state     = CDNS3_ROLE_STATE_INACTIVE;
2737         rdrv->name      = "gadget";
2738         cdns->roles[USB_ROLE_DEVICE] = rdrv;
2739
2740         return 0;
2741 }
2742
2743 /**
2744  * cdns3_gadget_uboot_handle_interrupt - handle cdns3 gadget interrupt
2745  * @cdns: pointer to struct cdns3
2746  *
2747  * Handles ep0 and gadget interrupt
2748  */
2749 static void cdns3_gadget_uboot_handle_interrupt(struct cdns3 *cdns)
2750 {
2751         int ret = cdns3_device_irq_handler(0, cdns);
2752
2753         if (ret == IRQ_WAKE_THREAD)
2754                 cdns3_device_thread_irq_handler(0, cdns);
2755 }
2756
2757 int dm_usb_gadget_handle_interrupts(struct udevice *dev)
2758 {
2759         struct cdns3 *cdns = dev_get_priv(dev);
2760
2761         cdns3_gadget_uboot_handle_interrupt(cdns);
2762
2763         return 0;
2764 }