drivers/usb/cdns3/gadget.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Cadence USBSS DRD Driver - gadget side.
   4  *
   5  * Copyright (C) 2018-2019 Cadence Design Systems.
   6  * Copyright (C) 2017-2018 NXP
   7  *
   8  * Authors: Pawel Jez <pjez@cadence.com>,
   9  *          Pawel Laszczak <pawell@cadence.com>
  10  *          Peter Chen <peter.chen@nxp.com>
  11  */
  12
  13 /*
  14  * Work around 1:
  15  * At some situations, the controller may get stale data address in TRB
  16  * at below sequences:
  17  * 1. Controller read TRB includes data address
  18  * 2. Software updates TRBs includes data address and Cycle bit
  19  * 3. Controller read TRB which includes Cycle bit
  20  * 4. DMA run with stale data address
  21  *
  22  * To fix this problem, driver needs to make the first TRB in TD as invalid.
  23  * After preparing all TRBs driver needs to check the position of DMA and
  24  * if the DMA point to the first just added TRB and doorbell is 1,
  25  * then driver must defer making this TRB as valid. This TRB will be make
  26  * as valid during adding next TRB only if DMA is stopped or at TRBERR
  27  * interrupt.
  28  *
  29  * Issue has been fixed in DEV_VER_V3 version of controller.
  30  *
  31  * Work around 2:
  32  * Controller for OUT endpoints has shared on-chip buffers for all incoming
  33  * packets, including ep0out. It's FIFO buffer, so packets must be handle by DMA
  34  * in correct order. If the first packet in the buffer will not be handled,
  35  * then the following packets directed for other endpoints and  functions
  36  * will be blocked.
  37  * Additionally the packets directed to one endpoint can block entire on-chip
  38  * buffers. In this case transfer to other endpoints also will blocked.
  39  *
  40  * To resolve this issue after raising the descriptor missing interrupt
  41  * driver prepares internal usb_request object and use it to arm DMA transfer.
  42  *
  43  * The problematic situation was observed in case when endpoint has been enabled
  44  * but no usb_request were queued. Driver try detects such endpoints and will
  45  * use this workaround only for these endpoint.
  46  *
  47  * Driver use limited number of buffer. This number can be set by macro
  48  * CDNS3_WA2_NUM_BUFFERS.
  49  *
  50  * Such blocking situation was observed on ACM gadget. For this function
  51  * host send OUT data packet but ACM function is not prepared for this packet.
  52  * It's cause that buffer placed in on chip memory block transfer to other
  53  * endpoints.
  54  *
  55  * Issue has been fixed in DEV_VER_V2 version of controller.
  56  *
  57  */
  58
  59 #include <dm.h>
  60 #include <dm/device_compat.h>
  61 #include <dm/devres.h>
  62 #include <linux/bitops.h>
  63 #include <linux/delay.h>
  64 #include <linux/err.h>
  65 #include <linux/usb/gadget.h>
  66 #include <linux/compat.h>
  67 #include <linux/iopoll.h>
  68 #include <linux/dma-mapping.h>
  69 #include <linux/bitmap.h>
  70 #include <linux/bug.h>
  71
  72 #include "core.h"
  73 #include "gadget-export.h"
  74 #include "gadget.h"
  75 #include "trace.h"
  76 #include "drd.h"
  77
  78 #define readl_poll_timeout_atomic readl_poll_timeout
  79 #define usleep_range(a, b) udelay((b))
  80
  81 static int __cdns3_gadget_ep_queue(struct usb_ep *ep,
  82                                    struct usb_request *request,
  83                                    gfp_t gfp_flags);
  84
  85 /**
  86  * cdns3_set_register_bit - set bit in given register.
  87  * @ptr: address of device controller register to be read and changed
  88  * @mask: bits requested to set
  89  */
  90 void cdns3_set_register_bit(void __iomem *ptr, u32 mask)
  91 {
  92         mask = readl(ptr) | mask;
  93         writel(mask, ptr);
  94 }
  95
  96 /**
  97  * cdns3_ep_addr_to_index - Macro converts endpoint address to
  98  * index of endpoint object in cdns3_device.eps[] container
  99  * @ep_addr: endpoint address for which endpoint object is required
 100  *
 101  */
 102 u8 cdns3_ep_addr_to_index(u8 ep_addr)
 103 {
 104         return (((ep_addr & 0x7F)) + ((ep_addr & USB_DIR_IN) ? 16 : 0));
 105 }
 106
 107 static int cdns3_get_dma_pos(struct cdns3_device *priv_dev,
 108                              struct cdns3_endpoint *priv_ep)
 109 {
 110         int dma_index;
 111
 112         dma_index = readl(&priv_dev->regs->ep_traddr) - priv_ep->trb_pool_dma;
 113
 114         return dma_index / TRB_SIZE;
 115 }
 116
 117 /**
 118  * cdns3_next_request - returns next request from list
 119  * @list: list containing requests
 120  *
 121  * Returns request or NULL if no requests in list
 122  */
 123 struct usb_request *cdns3_next_request(struct list_head *list)
 124 {
 125         return list_first_entry_or_null(list, struct usb_request, list);
 126 }
 127
 128 /**
 129  * cdns3_next_align_buf - returns next buffer from list
 130  * @list: list containing buffers
 131  *
 132  * Returns buffer or NULL if no buffers in list
 133  */
 134 struct cdns3_aligned_buf *cdns3_next_align_buf(struct list_head *list)
 135 {
 136         return list_first_entry_or_null(list, struct cdns3_aligned_buf, list);
 137 }
 138
 139 /**
 140  * cdns3_next_priv_request - returns next request from list
 141  * @list: list containing requests
 142  *
 143  * Returns request or NULL if no requests in list
 144  */
 145 struct cdns3_request *cdns3_next_priv_request(struct list_head *list)
 146 {
 147         return list_first_entry_or_null(list, struct cdns3_request, list);
 148 }
 149
 150 /**
 151  * select_ep - selects endpoint
 152  * @priv_dev:  extended gadget object
 153  * @ep: endpoint address
 154  */
 155 void cdns3_select_ep(struct cdns3_device *priv_dev, u32 ep)
 156 {
 157         if (priv_dev->selected_ep == ep)
 158                 return;
 159
 160         priv_dev->selected_ep = ep;
 161         writel(ep, &priv_dev->regs->ep_sel);
 162 }
 163
 164 dma_addr_t cdns3_trb_virt_to_dma(struct cdns3_endpoint *priv_ep,
 165                                  struct cdns3_trb *trb)
 166 {
 167         u32 offset = (char *)trb - (char *)priv_ep->trb_pool;
 168
 169         return priv_ep->trb_pool_dma + offset;
 170 }
 171
 172 int cdns3_ring_size(struct cdns3_endpoint *priv_ep)
 173 {
 174         switch (priv_ep->type) {
 175         case USB_ENDPOINT_XFER_ISOC:
 176                 return TRB_ISO_RING_SIZE;
 177         case USB_ENDPOINT_XFER_CONTROL:
 178                 return TRB_CTRL_RING_SIZE;
 179         default:
 180                 return TRB_RING_SIZE;
 181         }
 182 }
 183
 184 /**
 185  * cdns3_allocate_trb_pool - Allocates TRB's pool for selected endpoint
 186  * @priv_ep:  endpoint object
 187  *
 188  * Function will return 0 on success or -ENOMEM on allocation error
 189  */
 190 int cdns3_allocate_trb_pool(struct cdns3_endpoint *priv_ep)
 191 {
 192         int ring_size = cdns3_ring_size(priv_ep);
 193         struct cdns3_trb *link_trb;
 194
 195         if (!priv_ep->trb_pool) {
 196                 priv_ep->trb_pool =
 197                 dma_alloc_coherent(ring_size,
 198                                    (unsigned long *)&priv_ep->trb_pool_dma);
 199                 if (!priv_ep->trb_pool)
 200                         return -ENOMEM;
 201         } else {
 202                 memset(priv_ep->trb_pool, 0, ring_size);
 203         }
 204
 205         if (!priv_ep->num)
 206                 return 0;
 207
 208         priv_ep->num_trbs = ring_size / TRB_SIZE;
 209         /* Initialize the last TRB as Link TRB. */
 210         link_trb = (priv_ep->trb_pool + (priv_ep->num_trbs - 1));
 211         link_trb->buffer = TRB_BUFFER(priv_ep->trb_pool_dma);
 212         link_trb->control = TRB_CYCLE | TRB_TYPE(TRB_LINK) | TRB_TOGGLE;
 213
 214         return 0;
 215 }
 216
 217 static void cdns3_free_trb_pool(struct cdns3_endpoint *priv_ep)
 218 {
 219         if (priv_ep->trb_pool) {
 220                 dma_free_coherent(priv_ep->trb_pool);
 221                 priv_ep->trb_pool = NULL;
 222         }
 223 }
 224
 225 /**
 226  * cdns3_ep_stall_flush - Stalls and flushes selected endpoint
 227  * @priv_ep: endpoint object
 228  *
 229  * Endpoint must be selected before call to this function
 230  */
 231 static void cdns3_ep_stall_flush(struct cdns3_endpoint *priv_ep)
 232 {
 233         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
 234         int val;
 235
 236         trace_cdns3_halt(priv_ep, 1, 1);
 237
 238         writel(EP_CMD_DFLUSH | EP_CMD_ERDY | EP_CMD_SSTALL,
 239                &priv_dev->regs->ep_cmd);
 240
 241         /* wait for DFLUSH cleared */
 242         readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
 243                                   !(val & EP_CMD_DFLUSH), 1000);
 244         priv_ep->flags |= EP_STALLED;
 245         priv_ep->flags &= ~EP_STALL_PENDING;
 246 }
 247
 248 /**
 249  * cdns3_hw_reset_eps_config - reset endpoints configuration kept by controller.
 250  * @priv_dev: extended gadget object
 251  */
 252 void cdns3_hw_reset_eps_config(struct cdns3_device *priv_dev)
 253 {
 254         writel(USB_CONF_CFGRST, &priv_dev->regs->usb_conf);
 255
 256         cdns3_allow_enable_l1(priv_dev, 0);
 257         priv_dev->hw_configured_flag = 0;
 258         priv_dev->onchip_used_size = 0;
 259         priv_dev->out_mem_is_allocated = 0;
 260         priv_dev->wait_for_setup = 0;
 261 }
 262
 263 /**
 264  * cdns3_ep_inc_trb - increment a trb index.
 265  * @index: Pointer to the TRB index to increment.
 266  * @cs: Cycle state
 267  * @trb_in_seg: number of TRBs in segment
 268  *
 269  * The index should never point to the link TRB. After incrementing,
 270  * if it is point to the link TRB, wrap around to the beginning and revert
 271  * cycle state bit The
 272  * link TRB is always at the last TRB entry.
 273  */
 274 static void cdns3_ep_inc_trb(int *index, u8 *cs, int trb_in_seg)
 275 {
 276         (*index)++;
 277         if (*index == (trb_in_seg - 1)) {
 278                 *index = 0;
 279                 *cs ^=  1;
 280         }
 281 }
 282
 283 /**
 284  * cdns3_ep_inc_enq - increment endpoint's enqueue pointer
 285  * @priv_ep: The endpoint whose enqueue pointer we're incrementing
 286  */
 287 static void cdns3_ep_inc_enq(struct cdns3_endpoint *priv_ep)
 288 {
 289         priv_ep->free_trbs--;
 290         cdns3_ep_inc_trb(&priv_ep->enqueue, &priv_ep->pcs, priv_ep->num_trbs);
 291 }
 292
 293 /**
 294  * cdns3_ep_inc_deq - increment endpoint's dequeue pointer
 295  * @priv_ep: The endpoint whose dequeue pointer we're incrementing
 296  */
 297 static void cdns3_ep_inc_deq(struct cdns3_endpoint *priv_ep)
 298 {
 299         priv_ep->free_trbs++;
 300         cdns3_ep_inc_trb(&priv_ep->dequeue, &priv_ep->ccs, priv_ep->num_trbs);
 301 }
 302
 303 void cdns3_move_deq_to_next_trb(struct cdns3_request *priv_req)
 304 {
 305         struct cdns3_endpoint *priv_ep = priv_req->priv_ep;
 306         int current_trb = priv_req->start_trb;
 307
 308         while (current_trb != priv_req->end_trb) {
 309                 cdns3_ep_inc_deq(priv_ep);
 310                 current_trb = priv_ep->dequeue;
 311         }
 312
 313         cdns3_ep_inc_deq(priv_ep);
 314 }
 315
 316 /**
 317  * cdns3_allow_enable_l1 - enable/disable permits to transition to L1.
 318  * @priv_dev: Extended gadget object
 319  * @enable: Enable/disable permit to transition to L1.
 320  *
 321  * If bit USB_CONF_L1EN is set and device receive Extended Token packet,
 322  * then controller answer with ACK handshake.
 323  * If bit USB_CONF_L1DS is set and device receive Extended Token packet,
 324  * then controller answer with NYET handshake.
 325  */
 326 void cdns3_allow_enable_l1(struct cdns3_device *priv_dev, int enable)
 327 {
 328         if (enable)
 329                 writel(USB_CONF_L1EN, &priv_dev->regs->usb_conf);
 330         else
 331                 writel(USB_CONF_L1DS, &priv_dev->regs->usb_conf);
 332 }
 333
 334 enum usb_device_speed cdns3_get_speed(struct cdns3_device *priv_dev)
 335 {
 336         u32 reg;
 337
 338         reg = readl(&priv_dev->regs->usb_sts);
 339
 340         if (DEV_SUPERSPEED(reg))
 341                 return USB_SPEED_SUPER;
 342         else if (DEV_HIGHSPEED(reg))
 343                 return USB_SPEED_HIGH;
 344         else if (DEV_FULLSPEED(reg))
 345                 return USB_SPEED_FULL;
 346         else if (DEV_LOWSPEED(reg))
 347                 return USB_SPEED_LOW;
 348         return USB_SPEED_UNKNOWN;
 349 }
 350
 351 /**
 352  * cdns3_start_all_request - add to ring all request not started
 353  * @priv_dev: Extended gadget object
 354  * @priv_ep: The endpoint for whom request will be started.
 355  *
 356  * Returns return ENOMEM if transfer ring i not enough TRBs to start
 357  *         all requests.
 358  */
 359 static int cdns3_start_all_request(struct cdns3_device *priv_dev,
 360                                    struct cdns3_endpoint *priv_ep)
 361 {
 362         struct usb_request *request;
 363         int ret = 0;
 364
 365         while (!list_empty(&priv_ep->deferred_req_list)) {
 366                 request = cdns3_next_request(&priv_ep->deferred_req_list);
 367
 368                 ret = cdns3_ep_run_transfer(priv_ep, request);
 369                 if (ret)
 370                         return ret;
 371
 372                 list_del(&request->list);
 373                 list_add_tail(&request->list,
 374                               &priv_ep->pending_req_list);
 375         }
 376
 377         priv_ep->flags &= ~EP_RING_FULL;
 378         return ret;
 379 }
 380
 381 /*
 382  * WA2: Set flag for all not ISOC OUT endpoints. If this flag is set
 383  * driver try to detect whether endpoint need additional internal
 384  * buffer for unblocking on-chip FIFO buffer. This flag will be cleared
 385  * if before first DESCMISS interrupt the DMA will be armed.
 386  */
 387 #define cdns3_wa2_enable_detection(priv_dev, ep_priv, reg) do { \
 388         if (!priv_ep->dir && priv_ep->type != USB_ENDPOINT_XFER_ISOC) { \
 389                 priv_ep->flags |= EP_QUIRK_EXTRA_BUF_DET; \
 390                 (reg) |= EP_STS_EN_DESCMISEN; \
 391         } } while (0)
 392
 393 /**
 394  * cdns3_wa2_descmiss_copy_data copy data from internal requests to
 395  * request queued by class driver.
 396  * @priv_ep: extended endpoint object
 397  * @request: request object
 398  */
 399 static void cdns3_wa2_descmiss_copy_data(struct cdns3_endpoint *priv_ep,
 400                                          struct usb_request *request)
 401 {
 402         struct usb_request *descmiss_req;
 403         struct cdns3_request *descmiss_priv_req;
 404
 405         while (!list_empty(&priv_ep->wa2_descmiss_req_list)) {
 406                 int chunk_end;
 407                 int length;
 408
 409                 descmiss_priv_req =
 410                         cdns3_next_priv_request(&priv_ep->wa2_descmiss_req_list);
 411                 descmiss_req = &descmiss_priv_req->request;
 412
 413                 /* driver can't touch pending request */
 414                 if (descmiss_priv_req->flags & REQUEST_PENDING)
 415                         break;
 416
 417                 chunk_end = descmiss_priv_req->flags & REQUEST_INTERNAL_CH;
 418                 length = request->actual + descmiss_req->actual;
 419
 420                 request->status = descmiss_req->status;
 421
 422                 if (length <= request->length) {
 423                         memcpy(&((u8 *)request->buf)[request->actual],
 424                                descmiss_req->buf,
 425                                descmiss_req->actual);
 426                         request->actual = length;
 427                 } else {
 428                         /* It should never occur */
 429                         request->status = -ENOMEM;
 430                 }
 431
 432                 list_del_init(&descmiss_priv_req->list);
 433
 434                 kfree(descmiss_req->buf);
 435                 cdns3_gadget_ep_free_request(&priv_ep->endpoint, descmiss_req);
 436                 --priv_ep->wa2_counter;
 437
 438                 if (!chunk_end)
 439                         break;
 440         }
 441 }
 442
 443 struct usb_request *cdns3_wa2_gadget_giveback(struct cdns3_device *priv_dev,
 444                                               struct cdns3_endpoint *priv_ep,
 445                                               struct cdns3_request *priv_req)
 446 {
 447         if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN &&
 448             priv_req->flags & REQUEST_INTERNAL) {
 449                 struct usb_request *req;
 450
 451                 req = cdns3_next_request(&priv_ep->deferred_req_list);
 452
 453                 priv_ep->descmis_req = NULL;
 454
 455                 if (!req)
 456                         return NULL;
 457
 458                 cdns3_wa2_descmiss_copy_data(priv_ep, req);
 459                 if (!(priv_ep->flags & EP_QUIRK_END_TRANSFER) &&
 460                     req->length != req->actual) {
 461                         /* wait for next part of transfer */
 462                         return NULL;
 463                 }
 464
 465                 if (req->status == -EINPROGRESS)
 466                         req->status = 0;
 467
 468                 list_del_init(&req->list);
 469                 cdns3_start_all_request(priv_dev, priv_ep);
 470                 return req;
 471         }
 472
 473         return &priv_req->request;
 474 }
 475
 476 int cdns3_wa2_gadget_ep_queue(struct cdns3_device *priv_dev,
 477                               struct cdns3_endpoint *priv_ep,
 478                               struct cdns3_request *priv_req)
 479 {
 480         int deferred = 0;
 481
 482         /*
 483          * If transfer was queued before DESCMISS appear than we
 484          * can disable handling of DESCMISS interrupt. Driver assumes that it
 485          * can disable special treatment for this endpoint.
 486          */
 487         if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_DET) {
 488                 u32 reg;
 489
 490                 cdns3_select_ep(priv_dev, priv_ep->num | priv_ep->dir);
 491                 priv_ep->flags &= ~EP_QUIRK_EXTRA_BUF_DET;
 492                 reg = readl(&priv_dev->regs->ep_sts_en);
 493                 reg &= ~EP_STS_EN_DESCMISEN;
 494                 trace_cdns3_wa2(priv_ep, "workaround disabled\n");
 495                 writel(reg, &priv_dev->regs->ep_sts_en);
 496         }
 497
 498         if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN) {
 499                 u8 pending_empty = list_empty(&priv_ep->pending_req_list);
 500                 u8 descmiss_empty = list_empty(&priv_ep->wa2_descmiss_req_list);
 501
 502                 /*
 503                  *  DESCMISS transfer has been finished, so data will be
 504                  *  directly copied from internal allocated usb_request
 505                  *  objects.
 506                  */
 507                 if (pending_empty && !descmiss_empty &&
 508                     !(priv_req->flags & REQUEST_INTERNAL)) {
 509                         cdns3_wa2_descmiss_copy_data(priv_ep,
 510                                                      &priv_req->request);
 511
 512                         trace_cdns3_wa2(priv_ep, "get internal stored data");
 513
 514                         list_add_tail(&priv_req->request.list,
 515                                       &priv_ep->pending_req_list);
 516                         cdns3_gadget_giveback(priv_ep, priv_req,
 517                                               priv_req->request.status);
 518
 519                         /*
 520                          * Intentionally driver returns positive value as
 521                          * correct value. It informs that transfer has
 522                          * been finished.
 523                          */
 524                         return EINPROGRESS;
 525                 }
 526
 527                 /*
 528                  * Driver will wait for completion DESCMISS transfer,
 529                  * before starts new, not DESCMISS transfer.
 530                  */
 531                 if (!pending_empty && !descmiss_empty) {
 532                         trace_cdns3_wa2(priv_ep, "wait for pending transfer\n");
 533                         deferred = 1;
 534                 }
 535
 536                 if (priv_req->flags & REQUEST_INTERNAL)
 537                         list_add_tail(&priv_req->list,
 538                                       &priv_ep->wa2_descmiss_req_list);
 539         }
 540
 541         return deferred;
 542 }
 543
 544 static void cdns3_wa2_remove_old_request(struct cdns3_endpoint *priv_ep)
 545 {
 546         struct cdns3_request *priv_req;
 547
 548         while (!list_empty(&priv_ep->wa2_descmiss_req_list)) {
 549                 u8 chain;
 550
 551                 priv_req = cdns3_next_priv_request(&priv_ep->wa2_descmiss_req_list);
 552                 chain = !!(priv_req->flags & REQUEST_INTERNAL_CH);
 553
 554                 trace_cdns3_wa2(priv_ep, "removes eldest request");
 555
 556                 kfree(priv_req->request.buf);
 557                 cdns3_gadget_ep_free_request(&priv_ep->endpoint,
 558                                              &priv_req->request);
 559                 list_del_init(&priv_req->list);
 560                 --priv_ep->wa2_counter;
 561
 562                 if (!chain)
 563                         break;
 564         }
 565 }
 566
 567 /**
 568  * cdns3_wa2_descmissing_packet - handles descriptor missing event.
 569  * @priv_dev: extended gadget object
 570  *
 571  * This function is used only for WA2. For more information see Work around 2
 572  * description.
 573  */
 574 static void cdns3_wa2_descmissing_packet(struct cdns3_endpoint *priv_ep)
 575 {
 576         struct cdns3_request *priv_req;
 577         struct usb_request *request;
 578
 579         if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_DET) {
 580                 priv_ep->flags &= ~EP_QUIRK_EXTRA_BUF_DET;
 581                 priv_ep->flags |= EP_QUIRK_EXTRA_BUF_EN;
 582         }
 583
 584         trace_cdns3_wa2(priv_ep, "Description Missing detected\n");
 585
 586         if (priv_ep->wa2_counter >= CDNS3_WA2_NUM_BUFFERS)
 587                 cdns3_wa2_remove_old_request(priv_ep);
 588
 589         request = cdns3_gadget_ep_alloc_request(&priv_ep->endpoint,
 590                                                 GFP_ATOMIC);
 591         if (!request)
 592                 goto err;
 593
 594         priv_req = to_cdns3_request(request);
 595         priv_req->flags |= REQUEST_INTERNAL;
 596
 597         /* if this field is still assigned it indicate that transfer related
 598          * with this request has not been finished yet. Driver in this
 599          * case simply allocate next request and assign flag REQUEST_INTERNAL_CH
 600          * flag to previous one. It will indicate that current request is
 601          * part of the previous one.
 602          */
 603         if (priv_ep->descmis_req)
 604                 priv_ep->descmis_req->flags |= REQUEST_INTERNAL_CH;
 605
 606         priv_req->request.buf = kzalloc(CDNS3_DESCMIS_BUF_SIZE,
 607                                         GFP_ATOMIC);
 608         priv_ep->wa2_counter++;
 609
 610         if (!priv_req->request.buf) {
 611                 cdns3_gadget_ep_free_request(&priv_ep->endpoint, request);
 612                 goto err;
 613         }
 614
 615         priv_req->request.length = CDNS3_DESCMIS_BUF_SIZE;
 616         priv_ep->descmis_req = priv_req;
 617
 618         __cdns3_gadget_ep_queue(&priv_ep->endpoint,
 619                                 &priv_ep->descmis_req->request,
 620                                 GFP_ATOMIC);
 621
 622         return;
 623
 624 err:
 625         dev_err(priv_ep->cdns3_dev->dev,
 626                 "Failed: No sufficient memory for DESCMIS\n");
 627 }
 628
 629 /**
 630  * cdns3_gadget_giveback - call struct usb_request's ->complete callback
 631  * @priv_ep: The endpoint to whom the request belongs to
 632  * @priv_req: The request we're giving back
 633  * @status: completion code for the request
 634  *
 635  * Must be called with controller's lock held and interrupts disabled. This
 636  * function will unmap @req and call its ->complete() callback to notify upper
 637  * layers that it has completed.
 638  */
 639 void cdns3_gadget_giveback(struct cdns3_endpoint *priv_ep,
 640                            struct cdns3_request *priv_req,
 641                            int status)
 642 {
 643         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
 644         struct usb_request *request = &priv_req->request;
 645
 646         list_del_init(&request->list);
 647
 648         if (request->status == -EINPROGRESS)
 649                 request->status = status;
 650
 651         usb_gadget_unmap_request(&priv_dev->gadget, request,
 652                                  priv_ep->dir);
 653
 654         if ((priv_req->flags & REQUEST_UNALIGNED) &&
 655             priv_ep->dir == USB_DIR_OUT && !request->status)
 656                 memcpy(request->buf, priv_req->aligned_buf->buf,
 657                        request->length);
 658
 659         priv_req->flags &= ~(REQUEST_PENDING | REQUEST_UNALIGNED);
 660         trace_cdns3_gadget_giveback(priv_req);
 661
 662         if (priv_dev->dev_ver < DEV_VER_V2) {
 663                 request = cdns3_wa2_gadget_giveback(priv_dev, priv_ep,
 664                                                     priv_req);
 665                 if (!request)
 666                         return;
 667         }
 668
 669         if (request->complete) {
 670                 spin_unlock(&priv_dev->lock);
 671                 usb_gadget_giveback_request(&priv_ep->endpoint,
 672                                             request);
 673                 spin_lock(&priv_dev->lock);
 674         }
 675
 676         if (request->buf == priv_dev->zlp_buf)
 677                 cdns3_gadget_ep_free_request(&priv_ep->endpoint, request);
 678 }
 679
 680 void cdns3_wa1_restore_cycle_bit(struct cdns3_endpoint *priv_ep)
 681 {
 682         /* Work around for stale data address in TRB*/
 683         if (priv_ep->wa1_set) {
 684                 trace_cdns3_wa1(priv_ep, "restore cycle bit");
 685
 686                 priv_ep->wa1_set = 0;
 687                 priv_ep->wa1_trb_index = 0xFFFF;
 688                 if (priv_ep->wa1_cycle_bit) {
 689                         priv_ep->wa1_trb->control =
 690                                 priv_ep->wa1_trb->control | 0x1;
 691                 } else {
 692                         priv_ep->wa1_trb->control =
 693                                 priv_ep->wa1_trb->control & ~0x1;
 694                 }
 695         }
 696 }
 697
 698 static void cdns3_free_aligned_request_buf(struct cdns3_device *priv_dev)
 699 {
 700         struct cdns3_aligned_buf *buf, *tmp;
 701         unsigned long flags;
 702
 703         spin_lock_irqsave(&priv_dev->lock, flags);
 704
 705         list_for_each_entry_safe(buf, tmp, &priv_dev->aligned_buf_list, list) {
 706                 if (!buf->in_use) {
 707                         list_del(&buf->list);
 708
 709                         /*
 710                          * Re-enable interrupts to free DMA capable memory.
 711                          * Driver can't free this memory with disabled
 712                          * interrupts.
 713                          */
 714                         spin_unlock_irqrestore(&priv_dev->lock, flags);
 715                         dma_free_coherent(buf->buf);
 716                         kfree(buf);
 717                         spin_lock_irqsave(&priv_dev->lock, flags);
 718                 }
 719         }
 720
 721         spin_unlock_irqrestore(&priv_dev->lock, flags);
 722 }
 723
 724 static int cdns3_prepare_aligned_request_buf(struct cdns3_request *priv_req)
 725 {
 726         struct cdns3_endpoint *priv_ep = priv_req->priv_ep;
 727         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
 728         struct cdns3_aligned_buf *buf;
 729
 730         /* check if buffer is aligned to 8. */
 731         if (!((uintptr_t)priv_req->request.buf & 0x7))
 732                 return 0;
 733
 734         buf = priv_req->aligned_buf;
 735
 736         if (!buf || priv_req->request.length > buf->size) {
 737                 buf = kzalloc(sizeof(*buf), GFP_ATOMIC);
 738                 if (!buf)
 739                         return -ENOMEM;
 740
 741                 buf->size = priv_req->request.length;
 742
 743                 buf->buf = dma_alloc_coherent(buf->size,
 744                                               (unsigned long *)&buf->dma);
 745                 if (!buf->buf) {
 746                         kfree(buf);
 747                         return -ENOMEM;
 748                 }
 749
 750                 if (priv_req->aligned_buf) {
 751                         trace_cdns3_free_aligned_request(priv_req);
 752                         priv_req->aligned_buf->in_use = 0;
 753 #ifndef __UBOOT__
 754                         queue_work(system_freezable_wq,
 755                                    &priv_dev->aligned_buf_wq);
 756 #else
 757                         cdns3_free_aligned_request_buf(priv_dev);
 758 #endif
 759                 }
 760
 761                 buf->in_use = 1;
 762                 priv_req->aligned_buf = buf;
 763
 764                 list_add_tail(&buf->list,
 765                               &priv_dev->aligned_buf_list);
 766         }
 767
 768         if (priv_ep->dir == USB_DIR_IN) {
 769                 memcpy(buf->buf, priv_req->request.buf,
 770                        priv_req->request.length);
 771         }
 772
 773         priv_req->flags |= REQUEST_UNALIGNED;
 774         trace_cdns3_prepare_aligned_request(priv_req);
 775
 776         return 0;
 777 }
 778
 779 static int cdns3_wa1_update_guard(struct cdns3_endpoint *priv_ep,
 780                                   struct cdns3_trb *trb)
 781 {
 782         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
 783
 784         if (!priv_ep->wa1_set) {
 785                 u32 doorbell;
 786
 787                 doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY);
 788
 789                 if (doorbell) {
 790                         priv_ep->wa1_cycle_bit = priv_ep->pcs ? TRB_CYCLE : 0;
 791                         priv_ep->wa1_set = 1;
 792                         priv_ep->wa1_trb = trb;
 793                         priv_ep->wa1_trb_index = priv_ep->enqueue;
 794                         trace_cdns3_wa1(priv_ep, "set guard");
 795                         return 0;
 796                 }
 797         }
 798         return 1;
 799 }
 800
 801 static void cdns3_wa1_tray_restore_cycle_bit(struct cdns3_device *priv_dev,
 802                                              struct cdns3_endpoint *priv_ep)
 803 {
 804         int dma_index;
 805         u32 doorbell;
 806
 807         doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY);
 808         dma_index = cdns3_get_dma_pos(priv_dev, priv_ep);
 809
 810         if (!doorbell || dma_index != priv_ep->wa1_trb_index)
 811                 cdns3_wa1_restore_cycle_bit(priv_ep);
 812 }
 813
 814 /**
 815  * cdns3_ep_run_transfer - start transfer on no-default endpoint hardware
 816  * @priv_ep: endpoint object
 817  *
 818  * Returns zero on success or negative value on failure
 819  */
 820 int cdns3_ep_run_transfer(struct cdns3_endpoint *priv_ep,
 821                           struct usb_request *request)
 822 {
 823         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
 824         struct cdns3_request *priv_req;
 825         struct cdns3_trb *trb;
 826         dma_addr_t trb_dma;
 827         u32 togle_pcs = 1;
 828         int sg_iter = 0;
 829         int num_trb = 1;
 830         int address;
 831         u32 control;
 832         int pcs;
 833
 834         if (num_trb > priv_ep->free_trbs) {
 835                 priv_ep->flags |= EP_RING_FULL;
 836                 return -ENOBUFS;
 837         }
 838
 839         priv_req = to_cdns3_request(request);
 840         address = priv_ep->endpoint.desc->bEndpointAddress;
 841
 842         priv_ep->flags |= EP_PENDING_REQUEST;
 843
 844         /* must allocate buffer aligned to 8 */
 845         if (priv_req->flags & REQUEST_UNALIGNED)
 846                 trb_dma = priv_req->aligned_buf->dma;
 847         else
 848                 trb_dma = request->dma;
 849
 850         trb = priv_ep->trb_pool + priv_ep->enqueue;
 851         priv_req->start_trb = priv_ep->enqueue;
 852         priv_req->trb = trb;
 853
 854         cdns3_select_ep(priv_ep->cdns3_dev, address);
 855
 856         /* prepare ring */
 857         if ((priv_ep->enqueue + num_trb)  >= (priv_ep->num_trbs - 1)) {
 858                 struct cdns3_trb *link_trb;
 859                 int doorbell, dma_index;
 860                 u32 ch_bit = 0;
 861
 862                 doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY);
 863                 dma_index = cdns3_get_dma_pos(priv_dev, priv_ep);
 864
 865                 /* Driver can't update LINK TRB if it is current processed. */
 866                 if (doorbell && dma_index == priv_ep->num_trbs - 1) {
 867                         priv_ep->flags |= EP_DEFERRED_DRDY;
 868                         return -ENOBUFS;
 869                 }
 870
 871                 /*updating C bt in  Link TRB before starting DMA*/
 872                 link_trb = priv_ep->trb_pool + (priv_ep->num_trbs - 1);
 873                 /*
 874                  * For TRs size equal 2 enabling TRB_CHAIN for epXin causes
 875                  * that DMA stuck at the LINK TRB.
 876                  * On the other hand, removing TRB_CHAIN for longer TRs for
 877                  * epXout cause that DMA stuck after handling LINK TRB.
 878                  * To eliminate this strange behavioral driver set TRB_CHAIN
 879                  * bit only for TR size > 2.
 880                  */
 881                 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC ||
 882                     TRBS_PER_SEGMENT > 2)
 883                         ch_bit = TRB_CHAIN;
 884
 885                 link_trb->control = ((priv_ep->pcs) ? TRB_CYCLE : 0) |
 886                                     TRB_TYPE(TRB_LINK) | TRB_TOGGLE | ch_bit;
 887         }
 888
 889         if (priv_dev->dev_ver <= DEV_VER_V2)
 890                 togle_pcs = cdns3_wa1_update_guard(priv_ep, trb);
 891
 892         /* set incorrect Cycle Bit for first trb*/
 893         control = priv_ep->pcs ? 0 : TRB_CYCLE;
 894
 895         do {
 896                 u32 length;
 897                 u16 td_size = 0;
 898
 899                 /* fill TRB */
 900                 control |= TRB_TYPE(TRB_NORMAL);
 901                 trb->buffer = TRB_BUFFER(trb_dma);
 902
 903                 length = request->length;
 904
 905                 if (likely(priv_dev->dev_ver >= DEV_VER_V2))
 906                         td_size = DIV_ROUND_UP(length,
 907                                                priv_ep->endpoint.maxpacket);
 908
 909                 trb->length = TRB_BURST_LEN(priv_ep->trb_burst_size) |
 910                                         TRB_LEN(length);
 911                 if (priv_dev->gadget.speed == USB_SPEED_SUPER)
 912                         trb->length |= TRB_TDL_SS_SIZE(td_size);
 913                 else
 914                         control |= TRB_TDL_HS_SIZE(td_size);
 915
 916                 pcs = priv_ep->pcs ? TRB_CYCLE : 0;
 917
 918                 /*
 919                  * first trb should be prepared as last to avoid processing
 920                  *  transfer to early
 921                  */
 922                 if (sg_iter != 0)
 923                         control |= pcs;
 924
 925                 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC  && !priv_ep->dir) {
 926                         control |= TRB_IOC | TRB_ISP;
 927                 } else {
 928                         /* for last element in TD or in SG list */
 929                         if (sg_iter == (num_trb - 1) && sg_iter != 0)
 930                                 control |= pcs | TRB_IOC | TRB_ISP;
 931                 }
 932
 933                 if (sg_iter)
 934                         trb->control = control;
 935                 else
 936                         priv_req->trb->control = control;
 937
 938                 control = 0;
 939                 ++sg_iter;
 940                 priv_req->end_trb = priv_ep->enqueue;
 941                 cdns3_ep_inc_enq(priv_ep);
 942                 trb = priv_ep->trb_pool + priv_ep->enqueue;
 943         } while (sg_iter < num_trb);
 944
 945         trb = priv_req->trb;
 946
 947         priv_req->flags |= REQUEST_PENDING;
 948
 949         if (sg_iter == 1)
 950                 trb->control |= TRB_IOC | TRB_ISP;
 951
 952         /*
 953          * Memory barrier - cycle bit must be set before other filds in trb.
 954          */
 955         dmb();
 956
 957         /* give the TD to the consumer*/
 958         if (togle_pcs)
 959                 trb->control =  trb->control ^ 1;
 960
 961         if (priv_dev->dev_ver <= DEV_VER_V2)
 962                 cdns3_wa1_tray_restore_cycle_bit(priv_dev, priv_ep);
 963
 964         trace_cdns3_prepare_trb(priv_ep, priv_req->trb);
 965
 966         /*
 967          * Memory barrier - Cycle Bit must be set before trb->length  and
 968          * trb->buffer fields.
 969          */
 970         dmb();
 971
 972         /*
 973          * For DMULT mode we can set address to transfer ring only once after
 974          * enabling endpoint.
 975          */
 976         if (priv_ep->flags & EP_UPDATE_EP_TRBADDR) {
 977                 /*
 978                  * Until SW is not ready to handle the OUT transfer the ISO OUT
 979                  * Endpoint should be disabled (EP_CFG.ENABLE = 0).
 980                  * EP_CFG_ENABLE must be set before updating ep_traddr.
 981                  */
 982                 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC  && !priv_ep->dir &&
 983                     !(priv_ep->flags & EP_QUIRK_ISO_OUT_EN)) {
 984                         priv_ep->flags |= EP_QUIRK_ISO_OUT_EN;
 985                         cdns3_set_register_bit(&priv_dev->regs->ep_cfg,
 986                                                EP_CFG_ENABLE);
 987                 }
 988
 989                 writel(EP_TRADDR_TRADDR(priv_ep->trb_pool_dma +
 990                                         priv_req->start_trb * TRB_SIZE),
 991                                         &priv_dev->regs->ep_traddr);
 992
 993                 priv_ep->flags &= ~EP_UPDATE_EP_TRBADDR;
 994         }
 995
 996         if (!priv_ep->wa1_set && !(priv_ep->flags & EP_STALLED)) {
 997                 trace_cdns3_ring(priv_ep);
 998                 /*clearing TRBERR and EP_STS_DESCMIS before seting DRDY*/
 999                 writel(EP_STS_TRBERR | EP_STS_DESCMIS, &priv_dev->regs->ep_sts);
1000                 writel(EP_CMD_DRDY, &priv_dev->regs->ep_cmd);
1001                 trace_cdns3_doorbell_epx(priv_ep->name,
1002                                          readl(&priv_dev->regs->ep_traddr));
1003         }
1004
1005         /* WORKAROUND for transition to L0 */
1006         __cdns3_gadget_wakeup(priv_dev);
1007
1008         return 0;
1009 }
1010
1011 void cdns3_set_hw_configuration(struct cdns3_device *priv_dev)
1012 {
1013         struct cdns3_endpoint *priv_ep;
1014         struct usb_ep *ep;
1015         int val;
1016
1017         if (priv_dev->hw_configured_flag)
1018                 return;
1019
1020         writel(USB_CONF_CFGSET, &priv_dev->regs->usb_conf);
1021         writel(EP_CMD_ERDY | EP_CMD_REQ_CMPL, &priv_dev->regs->ep_cmd);
1022
1023         cdns3_set_register_bit(&priv_dev->regs->usb_conf,
1024                                USB_CONF_U1EN | USB_CONF_U2EN);
1025
1026         /* wait until configuration set */
1027         readl_poll_timeout_atomic(&priv_dev->regs->usb_sts, val,
1028                                   val & USB_STS_CFGSTS_MASK, 100);
1029
1030         priv_dev->hw_configured_flag = 1;
1031
1032         list_for_each_entry(ep, &priv_dev->gadget.ep_list, ep_list) {
1033                 priv_ep = ep_to_cdns3_ep(ep);
1034                 if (priv_ep->flags & EP_ENABLED)
1035                         cdns3_start_all_request(priv_dev, priv_ep);
1036         }
1037 }
1038
1039 /**
1040  * cdns3_request_handled - check whether request has been handled by DMA
1041  *
1042  * @priv_ep: extended endpoint object.
1043  * @priv_req: request object for checking
1044  *
1045  * Endpoint must be selected before invoking this function.
1046  *
1047  * Returns false if request has not been handled by DMA, else returns true.
1048  *
1049  * SR - start ring
1050  * ER -  end ring
1051  * DQ = priv_ep->dequeue - dequeue position
1052  * EQ = priv_ep->enqueue -  enqueue position
1053  * ST = priv_req->start_trb - index of first TRB in transfer ring
1054  * ET = priv_req->end_trb - index of last TRB in transfer ring
1055  * CI = current_index - index of processed TRB by DMA.
1056  *
1057  * As first step, function checks if cycle bit for priv_req->start_trb is
1058  * correct.
1059  *
1060  * some rules:
1061  * 1. priv_ep->dequeue never exceed current_index.
1062  * 2  priv_ep->enqueue never exceed priv_ep->dequeue
1063  * 3. exception: priv_ep->enqueue == priv_ep->dequeue
1064  *    and priv_ep->free_trbs is zero.
1065  *    This case indicate that TR is full.
1066  *
1067  * Then We can split recognition into two parts:
1068  * Case 1 - priv_ep->dequeue < current_index
1069  *      SR ... EQ ... DQ ... CI ... ER
1070  *      SR ... DQ ... CI ... EQ ... ER
1071  *
1072  *      Request has been handled by DMA if ST and ET is between DQ and CI.
1073  *
1074  * Case 2 - priv_ep->dequeue > current_index
1075  * This situation take place when CI go through the LINK TRB at the end of
1076  * transfer ring.
1077  *      SR ... CI ... EQ ... DQ ... ER
1078  *
1079  *      Request has been handled by DMA if ET is less then CI or
1080  *      ET is greater or equal DQ.
1081  */
1082 static bool cdns3_request_handled(struct cdns3_endpoint *priv_ep,
1083                                   struct cdns3_request *priv_req)
1084 {
1085         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1086         struct cdns3_trb *trb = priv_req->trb;
1087         int current_index = 0;
1088         int handled = 0;
1089         int doorbell;
1090
1091         current_index = cdns3_get_dma_pos(priv_dev, priv_ep);
1092         doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY);
1093
1094         trb = &priv_ep->trb_pool[priv_req->start_trb];
1095
1096         if ((trb->control  & TRB_CYCLE) != priv_ep->ccs)
1097                 goto finish;
1098
1099         if (doorbell == 1 && current_index == priv_ep->dequeue)
1100                 goto finish;
1101
1102         /* The corner case for TRBS_PER_SEGMENT equal 2). */
1103         if (TRBS_PER_SEGMENT == 2 && priv_ep->type != USB_ENDPOINT_XFER_ISOC) {
1104                 handled = 1;
1105                 goto finish;
1106         }
1107
1108         if (priv_ep->enqueue == priv_ep->dequeue &&
1109             priv_ep->free_trbs == 0) {
1110                 handled = 1;
1111         } else if (priv_ep->dequeue < current_index) {
1112                 if ((current_index == (priv_ep->num_trbs - 1)) &&
1113                     !priv_ep->dequeue)
1114                         goto finish;
1115
1116                 if (priv_req->end_trb >= priv_ep->dequeue &&
1117                     priv_req->end_trb < current_index)
1118                         handled = 1;
1119         } else if (priv_ep->dequeue  > current_index) {
1120                 if (priv_req->end_trb  < current_index ||
1121                     priv_req->end_trb >= priv_ep->dequeue)
1122                         handled = 1;
1123         }
1124
1125 finish:
1126         trace_cdns3_request_handled(priv_req, current_index, handled);
1127
1128         return handled;
1129 }
1130
1131 static void cdns3_transfer_completed(struct cdns3_device *priv_dev,
1132                                      struct cdns3_endpoint *priv_ep)
1133 {
1134         struct cdns3_request *priv_req;
1135         struct usb_request *request;
1136         struct cdns3_trb *trb;
1137
1138         while (!list_empty(&priv_ep->pending_req_list)) {
1139                 request = cdns3_next_request(&priv_ep->pending_req_list);
1140                 priv_req = to_cdns3_request(request);
1141
1142                 /* Re-select endpoint. It could be changed by other CPU during
1143                  * handling usb_gadget_giveback_request.
1144                  */
1145 #ifndef __UBOOT__
1146                 cdns3_select_ep(priv_dev, priv_ep->endpoint.address);
1147 #else
1148                 cdns3_select_ep(priv_dev,
1149                                 priv_ep->endpoint.desc->bEndpointAddress);
1150 #endif
1151
1152                 if (!cdns3_request_handled(priv_ep, priv_req))
1153                         goto prepare_next_td;
1154
1155                 trb = priv_ep->trb_pool + priv_ep->dequeue;
1156                 trace_cdns3_complete_trb(priv_ep, trb);
1157
1158                 if (trb != priv_req->trb)
1159                         dev_warn(priv_dev->dev,
1160                                  "request_trb=0x%p, queue_trb=0x%p\n",
1161                                  priv_req->trb, trb);
1162
1163                 request->actual = TRB_LEN(le32_to_cpu(trb->length));
1164                 cdns3_move_deq_to_next_trb(priv_req);
1165                 cdns3_gadget_giveback(priv_ep, priv_req, 0);
1166
1167                 if (priv_ep->type != USB_ENDPOINT_XFER_ISOC &&
1168                     TRBS_PER_SEGMENT == 2)
1169                         break;
1170         }
1171         priv_ep->flags &= ~EP_PENDING_REQUEST;
1172
1173 prepare_next_td:
1174         if (!(priv_ep->flags & EP_STALLED) &&
1175             !(priv_ep->flags & EP_STALL_PENDING))
1176                 cdns3_start_all_request(priv_dev, priv_ep);
1177 }
1178
1179 void cdns3_rearm_transfer(struct cdns3_endpoint *priv_ep, u8 rearm)
1180 {
1181         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1182
1183         cdns3_wa1_restore_cycle_bit(priv_ep);
1184
1185         if (rearm) {
1186                 trace_cdns3_ring(priv_ep);
1187
1188                 /* Cycle Bit must be updated before arming DMA. */
1189                 dmb();
1190                 writel(EP_CMD_DRDY, &priv_dev->regs->ep_cmd);
1191
1192                 __cdns3_gadget_wakeup(priv_dev);
1193
1194                 trace_cdns3_doorbell_epx(priv_ep->name,
1195                                          readl(&priv_dev->regs->ep_traddr));
1196         }
1197 }
1198
1199 /**
1200  * cdns3_check_ep_interrupt_proceed - Processes interrupt related to endpoint
1201  * @priv_ep: endpoint object
1202  *
1203  * Returns 0
1204  */
1205 static int cdns3_check_ep_interrupt_proceed(struct cdns3_endpoint *priv_ep)
1206 {
1207         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1208         u32 ep_sts_reg;
1209
1210 #ifndef __UBOOT__
1211         cdns3_select_ep(priv_dev, priv_ep->endpoint.address);
1212 #else
1213         cdns3_select_ep(priv_dev, priv_ep->endpoint.desc->bEndpointAddress);
1214 #endif
1215
1216         trace_cdns3_epx_irq(priv_dev, priv_ep);
1217
1218         ep_sts_reg = readl(&priv_dev->regs->ep_sts);
1219         writel(ep_sts_reg, &priv_dev->regs->ep_sts);
1220
1221         if (ep_sts_reg & EP_STS_TRBERR) {
1222                 if (priv_ep->flags & EP_STALL_PENDING &&
1223                     !(ep_sts_reg & EP_STS_DESCMIS &&
1224                     priv_dev->dev_ver < DEV_VER_V2)) {
1225                         cdns3_ep_stall_flush(priv_ep);
1226                 }
1227
1228                 /*
1229                  * For isochronous transfer driver completes request on
1230                  * IOC or on TRBERR. IOC appears only when device receive
1231                  * OUT data packet. If host disable stream or lost some packet
1232                  * then the only way to finish all queued transfer is to do it
1233                  * on TRBERR event.
1234                  */
1235                 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC &&
1236                     !priv_ep->wa1_set) {
1237                         if (!priv_ep->dir) {
1238                                 u32 ep_cfg = readl(&priv_dev->regs->ep_cfg);
1239
1240                                 ep_cfg &= ~EP_CFG_ENABLE;
1241                                 writel(ep_cfg, &priv_dev->regs->ep_cfg);
1242                                 priv_ep->flags &= ~EP_QUIRK_ISO_OUT_EN;
1243                         }
1244                         cdns3_transfer_completed(priv_dev, priv_ep);
1245                 } else if (!(priv_ep->flags & EP_STALLED) &&
1246                           !(priv_ep->flags & EP_STALL_PENDING)) {
1247                         if (priv_ep->flags & EP_DEFERRED_DRDY) {
1248                                 priv_ep->flags &= ~EP_DEFERRED_DRDY;
1249                                 cdns3_start_all_request(priv_dev, priv_ep);
1250                         } else {
1251                                 cdns3_rearm_transfer(priv_ep,
1252                                                      priv_ep->wa1_set);
1253                         }
1254                 }
1255         }
1256
1257         if ((ep_sts_reg & EP_STS_IOC) || (ep_sts_reg & EP_STS_ISP)) {
1258                 if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN) {
1259                         if (ep_sts_reg & EP_STS_ISP)
1260                                 priv_ep->flags |= EP_QUIRK_END_TRANSFER;
1261                         else
1262                                 priv_ep->flags &= ~EP_QUIRK_END_TRANSFER;
1263                 }
1264
1265                 cdns3_transfer_completed(priv_dev, priv_ep);
1266         }
1267
1268         /*
1269          * WA2: this condition should only be meet when
1270          * priv_ep->flags & EP_QUIRK_EXTRA_BUF_DET or
1271          * priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN.
1272          * In other cases this interrupt will be disabled/
1273          */
1274         if (ep_sts_reg & EP_STS_DESCMIS && priv_dev->dev_ver < DEV_VER_V2 &&
1275             !(priv_ep->flags & EP_STALLED))
1276                 cdns3_wa2_descmissing_packet(priv_ep);
1277
1278         return 0;
1279 }
1280
1281 static void cdns3_disconnect_gadget(struct cdns3_device *priv_dev)
1282 {
1283         if (priv_dev->gadget_driver && priv_dev->gadget_driver->disconnect) {
1284                 spin_unlock(&priv_dev->lock);
1285                 priv_dev->gadget_driver->disconnect(&priv_dev->gadget);
1286                 spin_lock(&priv_dev->lock);
1287         }
1288 }
1289
1290 /**
1291  * cdns3_check_usb_interrupt_proceed - Processes interrupt related to device
1292  * @priv_dev: extended gadget object
1293  * @usb_ists: bitmap representation of device's reported interrupts
1294  * (usb_ists register value)
1295  */
1296 static void cdns3_check_usb_interrupt_proceed(struct cdns3_device *priv_dev,
1297                                               u32 usb_ists)
1298 {
1299         int speed = 0;
1300
1301         trace_cdns3_usb_irq(priv_dev, usb_ists);
1302         if (usb_ists & USB_ISTS_L1ENTI) {
1303                 /*
1304                  * WORKAROUND: CDNS3 controller has issue with hardware resuming
1305                  * from L1. To fix it, if any DMA transfer is pending driver
1306                  * must starts driving resume signal immediately.
1307                  */
1308                 if (readl(&priv_dev->regs->drbl))
1309                         __cdns3_gadget_wakeup(priv_dev);
1310         }
1311
1312         /* Connection detected */
1313         if (usb_ists & (USB_ISTS_CON2I | USB_ISTS_CONI)) {
1314                 speed = cdns3_get_speed(priv_dev);
1315                 priv_dev->gadget.speed = speed;
1316                 usb_gadget_set_state(&priv_dev->gadget, USB_STATE_POWERED);
1317                 cdns3_ep0_config(priv_dev);
1318         }
1319
1320         /* Disconnection detected */
1321         if (usb_ists & (USB_ISTS_DIS2I | USB_ISTS_DISI)) {
1322                 cdns3_disconnect_gadget(priv_dev);
1323                 priv_dev->gadget.speed = USB_SPEED_UNKNOWN;
1324                 usb_gadget_set_state(&priv_dev->gadget, USB_STATE_NOTATTACHED);
1325                 cdns3_hw_reset_eps_config(priv_dev);
1326         }
1327
1328         if (usb_ists & (USB_ISTS_L2ENTI | USB_ISTS_U3ENTI)) {
1329                 if (priv_dev->gadget_driver &&
1330                     priv_dev->gadget_driver->suspend) {
1331                         spin_unlock(&priv_dev->lock);
1332                         priv_dev->gadget_driver->suspend(&priv_dev->gadget);
1333                         spin_lock(&priv_dev->lock);
1334                 }
1335         }
1336
1337         if (usb_ists & (USB_ISTS_L2EXTI | USB_ISTS_U3EXTI)) {
1338                 if (priv_dev->gadget_driver &&
1339                     priv_dev->gadget_driver->resume) {
1340                         spin_unlock(&priv_dev->lock);
1341                         priv_dev->gadget_driver->resume(&priv_dev->gadget);
1342                         spin_lock(&priv_dev->lock);
1343                 }
1344         }
1345
1346         /* reset*/
1347         if (usb_ists & (USB_ISTS_UWRESI | USB_ISTS_UHRESI | USB_ISTS_U2RESI)) {
1348                 if (priv_dev->gadget_driver) {
1349                         spin_unlock(&priv_dev->lock);
1350                         usb_gadget_udc_reset(&priv_dev->gadget,
1351                                              priv_dev->gadget_driver);
1352                         spin_lock(&priv_dev->lock);
1353
1354                         /*read again to check the actual speed*/
1355                         speed = cdns3_get_speed(priv_dev);
1356                         priv_dev->gadget.speed = speed;
1357                         cdns3_hw_reset_eps_config(priv_dev);
1358                         cdns3_ep0_config(priv_dev);
1359                 }
1360         }
1361 }
1362
1363 /**
1364  * cdns3_device_irq_handler- interrupt handler for device part of controller
1365  *
1366  * @irq: irq number for cdns3 core device
1367  * @data: structure of cdns3
1368  *
1369  * Returns IRQ_HANDLED or IRQ_NONE
1370  */
1371 static irqreturn_t cdns3_device_irq_handler(int irq, void *data)
1372 {
1373         struct cdns3_device *priv_dev;
1374         struct cdns3 *cdns = data;
1375         irqreturn_t ret = IRQ_NONE;
1376         u32 reg;
1377
1378         priv_dev = cdns->gadget_dev;
1379
1380         /* check USB device interrupt */
1381         reg = readl(&priv_dev->regs->usb_ists);
1382         if (reg) {
1383                 /* After masking interrupts the new interrupts won't be
1384                  * reported in usb_ists/ep_ists. In order to not lose some
1385                  * of them driver disables only detected interrupts.
1386                  * They will be enabled ASAP after clearing source of
1387                  * interrupt. This an unusual behavior only applies to
1388                  * usb_ists register.
1389                  */
1390                 reg = ~reg & readl(&priv_dev->regs->usb_ien);
1391                 /* mask deferred interrupt. */
1392                 writel(reg, &priv_dev->regs->usb_ien);
1393                 ret = IRQ_WAKE_THREAD;
1394         }
1395
1396         /* check endpoint interrupt */
1397         reg = readl(&priv_dev->regs->ep_ists);
1398         if (reg) {
1399                 writel(0, &priv_dev->regs->ep_ien);
1400                 ret = IRQ_WAKE_THREAD;
1401         }
1402
1403         return ret;
1404 }
1405
1406 /**
1407  * cdns3_device_thread_irq_handler- interrupt handler for device part
1408  * of controller
1409  *
1410  * @irq: irq number for cdns3 core device
1411  * @data: structure of cdns3
1412  *
1413  * Returns IRQ_HANDLED or IRQ_NONE
1414  */
1415 static irqreturn_t cdns3_device_thread_irq_handler(int irq, void *data)
1416 {
1417         struct cdns3_device *priv_dev;
1418         struct cdns3 *cdns = data;
1419         irqreturn_t ret = IRQ_NONE;
1420         unsigned long flags;
1421         int bit;
1422         u32 reg;
1423
1424         priv_dev = cdns->gadget_dev;
1425         spin_lock_irqsave(&priv_dev->lock, flags);
1426
1427         reg = readl(&priv_dev->regs->usb_ists);
1428         if (reg) {
1429                 writel(reg, &priv_dev->regs->usb_ists);
1430                 writel(USB_IEN_INIT, &priv_dev->regs->usb_ien);
1431                 cdns3_check_usb_interrupt_proceed(priv_dev, reg);
1432                 ret = IRQ_HANDLED;
1433         }
1434
1435         reg = readl(&priv_dev->regs->ep_ists);
1436
1437         /* handle default endpoint OUT */
1438         if (reg & EP_ISTS_EP_OUT0) {
1439                 cdns3_check_ep0_interrupt_proceed(priv_dev, USB_DIR_OUT);
1440                 ret = IRQ_HANDLED;
1441         }
1442
1443         /* handle default endpoint IN */
1444         if (reg & EP_ISTS_EP_IN0) {
1445                 cdns3_check_ep0_interrupt_proceed(priv_dev, USB_DIR_IN);
1446                 ret = IRQ_HANDLED;
1447         }
1448
1449         /* check if interrupt from non default endpoint, if no exit */
1450         reg &= ~(EP_ISTS_EP_OUT0 | EP_ISTS_EP_IN0);
1451         if (!reg)
1452                 goto irqend;
1453
1454         for_each_set_bit(bit, (unsigned long *)&reg,
1455                          sizeof(u32) * BITS_PER_BYTE) {
1456                 cdns3_check_ep_interrupt_proceed(priv_dev->eps[bit]);
1457                 ret = IRQ_HANDLED;
1458         }
1459
1460 irqend:
1461         writel(~0, &priv_dev->regs->ep_ien);
1462         spin_unlock_irqrestore(&priv_dev->lock, flags);
1463
1464         return ret;
1465 }
1466
1467 /**
1468  * cdns3_ep_onchip_buffer_reserve - Try to reserve onchip buf for EP
1469  *
1470  * The real reservation will occur during write to EP_CFG register,
1471  * this function is used to check if the 'size' reservation is allowed.
1472  *
1473  * @priv_dev: extended gadget object
1474  * @size: the size (KB) for EP would like to allocate
1475  * @is_in: endpoint direction
1476  *
1477  * Return 0 if the required size can met or negative value on failure
1478  */
1479 static int cdns3_ep_onchip_buffer_reserve(struct cdns3_device *priv_dev,
1480                                           int size, int is_in)
1481 {
1482         int remained;
1483
1484         /* 2KB are reserved for EP0*/
1485         remained = priv_dev->onchip_buffers - priv_dev->onchip_used_size - 2;
1486
1487         if (is_in) {
1488                 if (remained < size)
1489                         return -EPERM;
1490
1491                 priv_dev->onchip_used_size += size;
1492         } else {
1493                 int required;
1494
1495                 /**
1496                  *  ALL OUT EPs are shared the same chunk onchip memory, so
1497                  * driver checks if it already has assigned enough buffers
1498                  */
1499                 if (priv_dev->out_mem_is_allocated >= size)
1500                         return 0;
1501
1502                 required = size - priv_dev->out_mem_is_allocated;
1503
1504                 if (required > remained)
1505                         return -EPERM;
1506
1507                 priv_dev->out_mem_is_allocated += required;
1508                 priv_dev->onchip_used_size += required;
1509         }
1510
1511         return 0;
1512 }
1513
1514 void cdns3_configure_dmult(struct cdns3_device *priv_dev,
1515                            struct cdns3_endpoint *priv_ep)
1516 {
1517         struct cdns3_usb_regs __iomem *regs = priv_dev->regs;
1518
1519         /* For dev_ver > DEV_VER_V2 DMULT is configured per endpoint */
1520         if (priv_dev->dev_ver <= DEV_VER_V2)
1521                 writel(USB_CONF_DMULT, &regs->usb_conf);
1522
1523         if (priv_dev->dev_ver == DEV_VER_V2)
1524                 writel(USB_CONF2_EN_TDL_TRB, &regs->usb_conf2);
1525
1526         if (priv_dev->dev_ver >= DEV_VER_V3 && priv_ep) {
1527                 u32 mask;
1528
1529                 if (priv_ep->dir)
1530                         mask = BIT(priv_ep->num + 16);
1531                 else
1532                         mask = BIT(priv_ep->num);
1533
1534                 if (priv_ep->type != USB_ENDPOINT_XFER_ISOC) {
1535                         cdns3_set_register_bit(&regs->tdl_from_trb, mask);
1536                         cdns3_set_register_bit(&regs->tdl_beh, mask);
1537                         cdns3_set_register_bit(&regs->tdl_beh2, mask);
1538                         cdns3_set_register_bit(&regs->dma_adv_td, mask);
1539                 }
1540
1541                 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && !priv_ep->dir)
1542                         cdns3_set_register_bit(&regs->tdl_from_trb, mask);
1543
1544                 cdns3_set_register_bit(&regs->dtrans, mask);
1545         }
1546 }
1547
1548 /**
1549  * cdns3_ep_config Configure hardware endpoint
1550  * @priv_ep: extended endpoint object
1551  */
1552 void cdns3_ep_config(struct cdns3_endpoint *priv_ep)
1553 {
1554         bool is_iso_ep = (priv_ep->type == USB_ENDPOINT_XFER_ISOC);
1555         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1556         u32 bEndpointAddress = priv_ep->num | priv_ep->dir;
1557         u32 max_packet_size = 0;
1558         u8 maxburst = 0;
1559         u32 ep_cfg = 0;
1560         u8 buffering;
1561         u8 mult = 0;
1562         int ret;
1563
1564         buffering = CDNS3_EP_BUF_SIZE - 1;
1565
1566         cdns3_configure_dmult(priv_dev, priv_ep);
1567
1568         switch (priv_ep->type) {
1569         case USB_ENDPOINT_XFER_INT:
1570                 ep_cfg = EP_CFG_EPTYPE(USB_ENDPOINT_XFER_INT);
1571
1572                 if ((priv_dev->dev_ver == DEV_VER_V2 && !priv_ep->dir) ||
1573                     priv_dev->dev_ver > DEV_VER_V2)
1574                         ep_cfg |= EP_CFG_TDL_CHK;
1575                 break;
1576         case USB_ENDPOINT_XFER_BULK:
1577                 ep_cfg = EP_CFG_EPTYPE(USB_ENDPOINT_XFER_BULK);
1578
1579                 if ((priv_dev->dev_ver == DEV_VER_V2  && !priv_ep->dir) ||
1580                     priv_dev->dev_ver > DEV_VER_V2)
1581                         ep_cfg |= EP_CFG_TDL_CHK;
1582                 break;
1583         default:
1584                 ep_cfg = EP_CFG_EPTYPE(USB_ENDPOINT_XFER_ISOC);
1585                 mult = CDNS3_EP_ISO_HS_MULT - 1;
1586                 buffering = mult + 1;
1587         }
1588
1589         switch (priv_dev->gadget.speed) {
1590         case USB_SPEED_FULL:
1591                 max_packet_size = is_iso_ep ? 1023 : 64;
1592                 break;
1593         case USB_SPEED_HIGH:
1594                 max_packet_size = is_iso_ep ? 1024 : 512;
1595                 break;
1596         case USB_SPEED_SUPER:
1597                 /* It's limitation that driver assumes in driver. */
1598                 mult = 0;
1599                 max_packet_size = 1024;
1600                 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC) {
1601                         maxburst = CDNS3_EP_ISO_SS_BURST - 1;
1602                         buffering = (mult + 1) *
1603                                     (maxburst + 1);
1604
1605                         if (priv_ep->interval > 1)
1606                                 buffering++;
1607                 } else {
1608                         maxburst = CDNS3_EP_BUF_SIZE - 1;
1609                 }
1610                 break;
1611         default:
1612                 /* all other speed are not supported */
1613                 return;
1614         }
1615
1616         if (max_packet_size == 1024)
1617                 priv_ep->trb_burst_size = 128;
1618         else if (max_packet_size >= 512)
1619                 priv_ep->trb_burst_size = 64;
1620         else
1621                 priv_ep->trb_burst_size = 16;
1622
1623         ret = cdns3_ep_onchip_buffer_reserve(priv_dev, buffering + 1,
1624                                              !!priv_ep->dir);
1625         if (ret) {
1626                 dev_err(priv_dev->dev, "onchip mem is full, ep is invalid\n");
1627                 return;
1628         }
1629
1630         ep_cfg |= EP_CFG_MAXPKTSIZE(max_packet_size) |
1631                   EP_CFG_MULT(mult) |
1632                   EP_CFG_BUFFERING(buffering) |
1633                   EP_CFG_MAXBURST(maxburst);
1634
1635         cdns3_select_ep(priv_dev, bEndpointAddress);
1636         writel(ep_cfg, &priv_dev->regs->ep_cfg);
1637
1638         dev_dbg(priv_dev->dev, "Configure %s: with val %08x\n",
1639                 priv_ep->name, ep_cfg);
1640 }
1641
1642 /* Find correct direction for HW endpoint according to description */
1643 static int cdns3_ep_dir_is_correct(struct usb_endpoint_descriptor *desc,
1644                                    struct cdns3_endpoint *priv_ep)
1645 {
1646         return (priv_ep->endpoint.caps.dir_in && usb_endpoint_dir_in(desc)) ||
1647                (priv_ep->endpoint.caps.dir_out && usb_endpoint_dir_out(desc));
1648 }
1649
1650 static struct
1651 cdns3_endpoint *cdns3_find_available_ep(struct cdns3_device *priv_dev,
1652                                         struct usb_endpoint_descriptor *desc)
1653 {
1654         struct usb_ep *ep;
1655         struct cdns3_endpoint *priv_ep;
1656
1657         list_for_each_entry(ep, &priv_dev->gadget.ep_list, ep_list) {
1658                 unsigned long num;
1659                 /* ep name pattern likes epXin or epXout */
1660                 char c[2] = {ep->name[2], '\0'};
1661
1662                 num = simple_strtoul(c, NULL, 10);
1663
1664                 priv_ep = ep_to_cdns3_ep(ep);
1665                 if (cdns3_ep_dir_is_correct(desc, priv_ep)) {
1666                         if (!(priv_ep->flags & EP_CLAIMED)) {
1667                                 priv_ep->num  = num;
1668                                 return priv_ep;
1669                         }
1670                 }
1671         }
1672
1673         return ERR_PTR(-ENOENT);
1674 }
1675
1676 /*
1677  *  Cadence IP has one limitation that all endpoints must be configured
1678  * (Type & MaxPacketSize) before setting configuration through hardware
1679  * register, it means we can't change endpoints configuration after
1680  * set_configuration.
1681  *
1682  * This function set EP_CLAIMED flag which is added when the gadget driver
1683  * uses usb_ep_autoconfig to configure specific endpoint;
1684  * When the udc driver receives set_configurion request,
1685  * it goes through all claimed endpoints, and configure all endpoints
1686  * accordingly.
1687  *
1688  * At usb_ep_ops.enable/disable, we only enable and disable endpoint through
1689  * ep_cfg register which can be changed after set_configuration, and do
1690  * some software operation accordingly.
1691  */
1692 static struct
1693 usb_ep *cdns3_gadget_match_ep(struct usb_gadget *gadget,
1694                               struct usb_endpoint_descriptor *desc,
1695                               struct usb_ss_ep_comp_descriptor *comp_desc)
1696 {
1697         struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
1698         struct cdns3_endpoint *priv_ep;
1699         unsigned long flags;
1700
1701         priv_ep = cdns3_find_available_ep(priv_dev, desc);
1702         if (IS_ERR(priv_ep)) {
1703                 dev_err(priv_dev->dev, "no available ep\n");
1704                 return NULL;
1705         }
1706
1707         dev_dbg(priv_dev->dev, "match endpoint: %s\n", priv_ep->name);
1708
1709         spin_lock_irqsave(&priv_dev->lock, flags);
1710         priv_ep->endpoint.desc = desc;
1711         priv_ep->dir  = usb_endpoint_dir_in(desc) ? USB_DIR_IN : USB_DIR_OUT;
1712         priv_ep->type = usb_endpoint_type(desc);
1713         priv_ep->flags |= EP_CLAIMED;
1714         priv_ep->interval = desc->bInterval ? BIT(desc->bInterval - 1) : 0;
1715
1716         spin_unlock_irqrestore(&priv_dev->lock, flags);
1717         return &priv_ep->endpoint;
1718 }
1719
1720 /**
1721  * cdns3_gadget_ep_alloc_request Allocates request
1722  * @ep: endpoint object associated with request
1723  * @gfp_flags: gfp flags
1724  *
1725  * Returns allocated request address, NULL on allocation error
1726  */
1727 struct usb_request *cdns3_gadget_ep_alloc_request(struct usb_ep *ep,
1728                                                   gfp_t gfp_flags)
1729 {
1730         struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep);
1731         struct cdns3_request *priv_req;
1732
1733         priv_req = kzalloc(sizeof(*priv_req), gfp_flags);
1734         if (!priv_req)
1735                 return NULL;
1736
1737         priv_req->priv_ep = priv_ep;
1738
1739         trace_cdns3_alloc_request(priv_req);
1740         return &priv_req->request;
1741 }
1742
1743 /**
1744  * cdns3_gadget_ep_free_request Free memory occupied by request
1745  * @ep: endpoint object associated with request
1746  * @request: request to free memory
1747  */
1748 void cdns3_gadget_ep_free_request(struct usb_ep *ep,
1749                                   struct usb_request *request)
1750 {
1751         struct cdns3_request *priv_req = to_cdns3_request(request);
1752
1753         if (priv_req->aligned_buf)
1754                 priv_req->aligned_buf->in_use = 0;
1755
1756         trace_cdns3_free_request(priv_req);
1757         kfree(priv_req);
1758 }
1759
1760 /**
1761  * cdns3_gadget_ep_enable Enable endpoint
1762  * @ep: endpoint object
1763  * @desc: endpoint descriptor
1764  *
1765  * Returns 0 on success, error code elsewhere
1766  */
1767 static int cdns3_gadget_ep_enable(struct usb_ep *ep,
1768                                   const struct usb_endpoint_descriptor *desc)
1769 {
1770         struct cdns3_endpoint *priv_ep;
1771         struct cdns3_device *priv_dev;
1772         u32 reg = EP_STS_EN_TRBERREN;
1773         u32 bEndpointAddress;
1774         unsigned long flags;
1775         int enable = 1;
1776         int ret;
1777         int val;
1778
1779         priv_ep = ep_to_cdns3_ep(ep);
1780         priv_dev = priv_ep->cdns3_dev;
1781
1782         if (!ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT) {
1783                 dev_dbg(priv_dev->dev, "usbss: invalid parameters\n");
1784                 return -EINVAL;
1785         }
1786
1787         if (!desc->wMaxPacketSize) {
1788                 dev_err(priv_dev->dev, "usbss: missing wMaxPacketSize\n");
1789                 return -EINVAL;
1790         }
1791
1792         if (WARN_ON(priv_ep->flags & EP_ENABLED))
1793                 return 0;
1794
1795         spin_lock_irqsave(&priv_dev->lock, flags);
1796
1797         priv_ep->endpoint.desc = desc;
1798         priv_ep->type = usb_endpoint_type(desc);
1799         priv_ep->interval = desc->bInterval ? BIT(desc->bInterval - 1) : 0;
1800
1801         if (priv_ep->interval > ISO_MAX_INTERVAL &&
1802             priv_ep->type == USB_ENDPOINT_XFER_ISOC) {
1803                 dev_err(priv_dev->dev, "Driver is limited to %d period\n",
1804                         ISO_MAX_INTERVAL);
1805
1806                 ret =  -EINVAL;
1807                 goto exit;
1808         }
1809
1810         ret = cdns3_allocate_trb_pool(priv_ep);
1811
1812         if (ret)
1813                 goto exit;
1814
1815         bEndpointAddress = priv_ep->num | priv_ep->dir;
1816         cdns3_select_ep(priv_dev, bEndpointAddress);
1817
1818         trace_cdns3_gadget_ep_enable(priv_ep);
1819
1820         writel(EP_CMD_EPRST, &priv_dev->regs->ep_cmd);
1821
1822         ret = readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
1823                                         !(val & (EP_CMD_CSTALL | EP_CMD_EPRST)),
1824                                         1000);
1825
1826         if (unlikely(ret)) {
1827                 cdns3_free_trb_pool(priv_ep);
1828                 ret =  -EINVAL;
1829                 goto exit;
1830         }
1831
1832         /* enable interrupt for selected endpoint */
1833         cdns3_set_register_bit(&priv_dev->regs->ep_ien,
1834                                BIT(cdns3_ep_addr_to_index(bEndpointAddress)));
1835
1836         if (priv_dev->dev_ver < DEV_VER_V2)
1837                 cdns3_wa2_enable_detection(priv_dev, priv_ep, reg);
1838
1839         writel(reg, &priv_dev->regs->ep_sts_en);
1840
1841         /*
1842          * For some versions of controller at some point during ISO OUT traffic
1843          * DMA reads Transfer Ring for the EP which has never got doorbell.
1844          * This issue was detected only on simulation, but to avoid this issue
1845          * driver add protection against it. To fix it driver enable ISO OUT
1846          * endpoint before setting DRBL. This special treatment of ISO OUT
1847          * endpoints are recommended by controller specification.
1848          */
1849         if (priv_ep->type == USB_ENDPOINT_XFER_ISOC  && !priv_ep->dir)
1850                 enable = 0;
1851
1852         if (enable)
1853                 cdns3_set_register_bit(&priv_dev->regs->ep_cfg, EP_CFG_ENABLE);
1854
1855         ep->desc = desc;
1856         priv_ep->flags &= ~(EP_PENDING_REQUEST | EP_STALLED | EP_STALL_PENDING |
1857                             EP_QUIRK_ISO_OUT_EN | EP_QUIRK_EXTRA_BUF_EN);
1858         priv_ep->flags |= EP_ENABLED | EP_UPDATE_EP_TRBADDR;
1859         priv_ep->wa1_set = 0;
1860         priv_ep->enqueue = 0;
1861         priv_ep->dequeue = 0;
1862         reg = readl(&priv_dev->regs->ep_sts);
1863         priv_ep->pcs = !!EP_STS_CCS(reg);
1864         priv_ep->ccs = !!EP_STS_CCS(reg);
1865         /* one TRB is reserved for link TRB used in DMULT mode*/
1866         priv_ep->free_trbs = priv_ep->num_trbs - 1;
1867 exit:
1868         spin_unlock_irqrestore(&priv_dev->lock, flags);
1869
1870         return ret;
1871 }
1872
1873 /**
1874  * cdns3_gadget_ep_disable Disable endpoint
1875  * @ep: endpoint object
1876  *
1877  * Returns 0 on success, error code elsewhere
1878  */
1879 static int cdns3_gadget_ep_disable(struct usb_ep *ep)
1880 {
1881         struct cdns3_endpoint *priv_ep;
1882         struct cdns3_request *priv_req;
1883         struct cdns3_device *priv_dev;
1884         struct usb_request *request;
1885         unsigned long flags;
1886         int ret = 0;
1887         u32 ep_cfg;
1888         int val;
1889
1890         if (!ep) {
1891                 pr_err("usbss: invalid parameters\n");
1892                 return -EINVAL;
1893         }
1894
1895         priv_ep = ep_to_cdns3_ep(ep);
1896         priv_dev = priv_ep->cdns3_dev;
1897
1898         if (WARN_ON(!(priv_ep->flags & EP_ENABLED)))
1899                 return 0;
1900
1901         spin_lock_irqsave(&priv_dev->lock, flags);
1902
1903         trace_cdns3_gadget_ep_disable(priv_ep);
1904
1905         cdns3_select_ep(priv_dev, ep->desc->bEndpointAddress);
1906
1907         ep_cfg = readl(&priv_dev->regs->ep_cfg);
1908         ep_cfg &= ~EP_CFG_ENABLE;
1909         writel(ep_cfg, &priv_dev->regs->ep_cfg);
1910
1911         /**
1912          * Driver needs some time before resetting endpoint.
1913          * It need waits for clearing DBUSY bit or for timeout expired.
1914          * 10us is enough time for controller to stop transfer.
1915          */
1916         readl_poll_timeout_atomic(&priv_dev->regs->ep_sts, val,
1917                                   !(val & EP_STS_DBUSY), 10);
1918         writel(EP_CMD_EPRST, &priv_dev->regs->ep_cmd);
1919
1920         readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
1921                                   !(val & (EP_CMD_CSTALL | EP_CMD_EPRST)),
1922                                   1000);
1923         if (unlikely(ret))
1924                 dev_err(priv_dev->dev, "Timeout: %s resetting failed.\n",
1925                         priv_ep->name);
1926
1927         while (!list_empty(&priv_ep->pending_req_list)) {
1928                 request = cdns3_next_request(&priv_ep->pending_req_list);
1929
1930                 cdns3_gadget_giveback(priv_ep, to_cdns3_request(request),
1931                                       -ESHUTDOWN);
1932         }
1933
1934         while (!list_empty(&priv_ep->wa2_descmiss_req_list)) {
1935                 priv_req = cdns3_next_priv_request(&priv_ep->wa2_descmiss_req_list);
1936
1937                 kfree(priv_req->request.buf);
1938                 cdns3_gadget_ep_free_request(&priv_ep->endpoint,
1939                                              &priv_req->request);
1940                 list_del_init(&priv_req->list);
1941                 --priv_ep->wa2_counter;
1942         }
1943
1944         while (!list_empty(&priv_ep->deferred_req_list)) {
1945                 request = cdns3_next_request(&priv_ep->deferred_req_list);
1946
1947                 cdns3_gadget_giveback(priv_ep, to_cdns3_request(request),
1948                                       -ESHUTDOWN);
1949         }
1950
1951         priv_ep->descmis_req = NULL;
1952
1953         ep->desc = NULL;
1954         priv_ep->flags &= ~EP_ENABLED;
1955
1956         spin_unlock_irqrestore(&priv_dev->lock, flags);
1957
1958         return ret;
1959 }
1960
1961 /**
1962  * cdns3_gadget_ep_queue Transfer data on endpoint
1963  * @ep: endpoint object
1964  * @request: request object
1965  * @gfp_flags: gfp flags
1966  *
1967  * Returns 0 on success, error code elsewhere
1968  */
1969 static int __cdns3_gadget_ep_queue(struct usb_ep *ep,
1970                                    struct usb_request *request,
1971                                    gfp_t gfp_flags)
1972 {
1973         struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep);
1974         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1975         struct cdns3_request *priv_req;
1976         int ret = 0;
1977
1978         request->actual = 0;
1979         request->status = -EINPROGRESS;
1980         priv_req = to_cdns3_request(request);
1981         trace_cdns3_ep_queue(priv_req);
1982
1983         if (priv_dev->dev_ver < DEV_VER_V2) {
1984                 ret = cdns3_wa2_gadget_ep_queue(priv_dev, priv_ep,
1985                                                 priv_req);
1986
1987                 if (ret == EINPROGRESS)
1988                         return 0;
1989         }
1990
1991         ret = cdns3_prepare_aligned_request_buf(priv_req);
1992         if (ret < 0)
1993                 return ret;
1994
1995         ret = usb_gadget_map_request(&priv_dev->gadget, request,
1996                                      usb_endpoint_dir_in(ep->desc));
1997         if (ret)
1998                 return ret;
1999
2000         list_add_tail(&request->list, &priv_ep->deferred_req_list);
2001
2002         /*
2003          * If hardware endpoint configuration has not been set yet then
2004          * just queue request in deferred list. Transfer will be started in
2005          * cdns3_set_hw_configuration.
2006          */
2007         if (priv_dev->hw_configured_flag && !(priv_ep->flags & EP_STALLED) &&
2008             !(priv_ep->flags & EP_STALL_PENDING))
2009                 cdns3_start_all_request(priv_dev, priv_ep);
2010
2011         return 0;
2012 }
2013
2014 static int cdns3_gadget_ep_queue(struct usb_ep *ep, struct usb_request *request,
2015                                  gfp_t gfp_flags)
2016 {
2017         struct usb_request *zlp_request;
2018         struct cdns3_endpoint *priv_ep;
2019         struct cdns3_device *priv_dev;
2020         unsigned long flags;
2021         int ret;
2022
2023         if (!request || !ep)
2024                 return -EINVAL;
2025
2026         priv_ep = ep_to_cdns3_ep(ep);
2027         priv_dev = priv_ep->cdns3_dev;
2028
2029         spin_lock_irqsave(&priv_dev->lock, flags);
2030
2031         ret = __cdns3_gadget_ep_queue(ep, request, gfp_flags);
2032
2033         if (ret == 0 && request->zero && request->length &&
2034             (request->length % ep->maxpacket == 0)) {
2035                 struct cdns3_request *priv_req;
2036
2037                 zlp_request = cdns3_gadget_ep_alloc_request(ep, GFP_ATOMIC);
2038                 zlp_request->buf = priv_dev->zlp_buf;
2039                 zlp_request->length = 0;
2040
2041                 priv_req = to_cdns3_request(zlp_request);
2042                 priv_req->flags |= REQUEST_ZLP;
2043
2044                 dev_dbg(priv_dev->dev, "Queuing ZLP for endpoint: %s\n",
2045                         priv_ep->name);
2046                 ret = __cdns3_gadget_ep_queue(ep, zlp_request, gfp_flags);
2047         }
2048
2049         spin_unlock_irqrestore(&priv_dev->lock, flags);
2050         return ret;
2051 }
2052
2053 /**
2054  * cdns3_gadget_ep_dequeue Remove request from transfer queue
2055  * @ep: endpoint object associated with request
2056  * @request: request object
2057  *
2058  * Returns 0 on success, error code elsewhere
2059  */
2060 int cdns3_gadget_ep_dequeue(struct usb_ep *ep,
2061                             struct usb_request *request)
2062 {
2063         struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep);
2064         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
2065         struct usb_request *req, *req_temp;
2066         struct cdns3_request *priv_req;
2067         struct cdns3_trb *link_trb;
2068         unsigned long flags;
2069         int ret = 0;
2070
2071         if (!ep || !request || !ep->desc)
2072                 return -EINVAL;
2073
2074         spin_lock_irqsave(&priv_dev->lock, flags);
2075
2076         priv_req = to_cdns3_request(request);
2077
2078         trace_cdns3_ep_dequeue(priv_req);
2079
2080         cdns3_select_ep(priv_dev, ep->desc->bEndpointAddress);
2081
2082         list_for_each_entry_safe(req, req_temp, &priv_ep->pending_req_list,
2083                                  list) {
2084                 if (request == req)
2085                         goto found;
2086         }
2087
2088         list_for_each_entry_safe(req, req_temp, &priv_ep->deferred_req_list,
2089                                  list) {
2090                 if (request == req)
2091                         goto found;
2092         }
2093
2094         goto not_found;
2095
2096 found:
2097
2098         if (priv_ep->wa1_trb == priv_req->trb)
2099                 cdns3_wa1_restore_cycle_bit(priv_ep);
2100
2101         link_trb = priv_req->trb;
2102         cdns3_move_deq_to_next_trb(priv_req);
2103         cdns3_gadget_giveback(priv_ep, priv_req, -ECONNRESET);
2104
2105         /* Update ring */
2106         request = cdns3_next_request(&priv_ep->deferred_req_list);
2107         if (request) {
2108                 priv_req = to_cdns3_request(request);
2109
2110                 link_trb->buffer = TRB_BUFFER(priv_ep->trb_pool_dma +
2111                                               (priv_req->start_trb * TRB_SIZE));
2112                 link_trb->control = (link_trb->control & TRB_CYCLE) |
2113                                     TRB_TYPE(TRB_LINK) | TRB_CHAIN | TRB_TOGGLE;
2114         } else {
2115                 priv_ep->flags |= EP_UPDATE_EP_TRBADDR;
2116         }
2117
2118 not_found:
2119         spin_unlock_irqrestore(&priv_dev->lock, flags);
2120         return ret;
2121 }
2122
2123 /**
2124  * __cdns3_gadget_ep_set_halt Sets stall on selected endpoint
2125  * Should be called after acquiring spin_lock and selecting ep
2126  * @ep: endpoint object to set stall on.
2127  */
2128 void __cdns3_gadget_ep_set_halt(struct cdns3_endpoint *priv_ep)
2129 {
2130         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
2131
2132         trace_cdns3_halt(priv_ep, 1, 0);
2133
2134         if (!(priv_ep->flags & EP_STALLED)) {
2135                 u32 ep_sts_reg = readl(&priv_dev->regs->ep_sts);
2136
2137                 if (!(ep_sts_reg & EP_STS_DBUSY))
2138                         cdns3_ep_stall_flush(priv_ep);
2139                 else
2140                         priv_ep->flags |= EP_STALL_PENDING;
2141         }
2142 }
2143
2144 /**
2145  * __cdns3_gadget_ep_clear_halt Clears stall on selected endpoint
2146  * Should be called after acquiring spin_lock and selecting ep
2147  * @ep: endpoint object to clear stall on
2148  */
2149 int __cdns3_gadget_ep_clear_halt(struct cdns3_endpoint *priv_ep)
2150 {
2151         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
2152         struct usb_request *request;
2153         int ret = 0;
2154         int val;
2155
2156         trace_cdns3_halt(priv_ep, 0, 0);
2157
2158         writel(EP_CMD_CSTALL | EP_CMD_EPRST, &priv_dev->regs->ep_cmd);
2159
2160         /* wait for EPRST cleared */
2161         readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
2162                                   !(val & EP_CMD_EPRST), 100);
2163         if (ret)
2164                 return -EINVAL;
2165
2166         priv_ep->flags &= ~(EP_STALLED | EP_STALL_PENDING);
2167
2168         request = cdns3_next_request(&priv_ep->pending_req_list);
2169
2170         if (request)
2171                 cdns3_rearm_transfer(priv_ep, 1);
2172
2173         cdns3_start_all_request(priv_dev, priv_ep);
2174         return ret;
2175 }
2176
2177 /**
2178  * cdns3_gadget_ep_set_halt Sets/clears stall on selected endpoint
2179  * @ep: endpoint object to set/clear stall on
2180  * @value: 1 for set stall, 0 for clear stall
2181  *
2182  * Returns 0 on success, error code elsewhere
2183  */
2184 int cdns3_gadget_ep_set_halt(struct usb_ep *ep, int value)
2185 {
2186         struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep);
2187         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
2188         unsigned long flags;
2189         int ret = 0;
2190
2191         if (!(priv_ep->flags & EP_ENABLED))
2192                 return -EPERM;
2193
2194         spin_lock_irqsave(&priv_dev->lock, flags);
2195
2196         cdns3_select_ep(priv_dev, ep->desc->bEndpointAddress);
2197
2198         if (!value) {
2199                 priv_ep->flags &= ~EP_WEDGE;
2200                 ret = __cdns3_gadget_ep_clear_halt(priv_ep);
2201         } else {
2202                 __cdns3_gadget_ep_set_halt(priv_ep);
2203         }
2204
2205         spin_unlock_irqrestore(&priv_dev->lock, flags);
2206
2207         return ret;
2208 }
2209
2210 extern const struct usb_ep_ops cdns3_gadget_ep0_ops;
2211
2212 static const struct usb_ep_ops cdns3_gadget_ep_ops = {
2213         .enable = cdns3_gadget_ep_enable,
2214         .disable = cdns3_gadget_ep_disable,
2215         .alloc_request = cdns3_gadget_ep_alloc_request,
2216         .free_request = cdns3_gadget_ep_free_request,
2217         .queue = cdns3_gadget_ep_queue,
2218         .dequeue = cdns3_gadget_ep_dequeue,
2219         .set_halt = cdns3_gadget_ep_set_halt,
2220         .set_wedge = cdns3_gadget_ep_set_wedge,
2221 };
2222
2223 /**
2224  * cdns3_gadget_get_frame Returns number of actual ITP frame
2225  * @gadget: gadget object
2226  *
2227  * Returns number of actual ITP frame
2228  */
2229 static int cdns3_gadget_get_frame(struct usb_gadget *gadget)
2230 {
2231         struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2232
2233         return readl(&priv_dev->regs->usb_itpn);
2234 }
2235
2236 int __cdns3_gadget_wakeup(struct cdns3_device *priv_dev)
2237 {
2238         enum usb_device_speed speed;
2239
2240         speed = cdns3_get_speed(priv_dev);
2241
2242         if (speed >= USB_SPEED_SUPER)
2243                 return 0;
2244
2245         /* Start driving resume signaling to indicate remote wakeup. */
2246         writel(USB_CONF_LGO_L0, &priv_dev->regs->usb_conf);
2247
2248         return 0;
2249 }
2250
2251 static int cdns3_gadget_wakeup(struct usb_gadget *gadget)
2252 {
2253         struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2254         unsigned long flags;
2255         int ret = 0;
2256
2257         spin_lock_irqsave(&priv_dev->lock, flags);
2258         ret = __cdns3_gadget_wakeup(priv_dev);
2259         spin_unlock_irqrestore(&priv_dev->lock, flags);
2260         return ret;
2261 }
2262
2263 static int cdns3_gadget_set_selfpowered(struct usb_gadget *gadget,
2264                                         int is_selfpowered)
2265 {
2266         struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2267         unsigned long flags;
2268
2269         spin_lock_irqsave(&priv_dev->lock, flags);
2270         priv_dev->is_selfpowered = !!is_selfpowered;
2271         spin_unlock_irqrestore(&priv_dev->lock, flags);
2272         return 0;
2273 }
2274
2275 static int cdns3_gadget_pullup(struct usb_gadget *gadget, int is_on)
2276 {
2277         struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2278
2279         if (is_on)
2280                 writel(USB_CONF_DEVEN, &priv_dev->regs->usb_conf);
2281         else
2282                 writel(USB_CONF_DEVDS, &priv_dev->regs->usb_conf);
2283
2284         return 0;
2285 }
2286
2287 static void cdns3_gadget_config(struct cdns3_device *priv_dev)
2288 {
2289         struct cdns3_usb_regs __iomem *regs = priv_dev->regs;
2290         u32 reg;
2291
2292         cdns3_ep0_config(priv_dev);
2293
2294         /* enable interrupts for endpoint 0 (in and out) */
2295         writel(EP_IEN_EP_OUT0 | EP_IEN_EP_IN0, &regs->ep_ien);
2296
2297         /*
2298          * Driver needs to modify LFPS minimal U1 Exit time for DEV_VER_TI_V1
2299          * revision of controller.
2300          */
2301         if (priv_dev->dev_ver == DEV_VER_TI_V1) {
2302                 reg = readl(&regs->dbg_link1);
2303
2304                 reg &= ~DBG_LINK1_LFPS_MIN_GEN_U1_EXIT_MASK;
2305                 reg |= DBG_LINK1_LFPS_MIN_GEN_U1_EXIT(0x55) |
2306                        DBG_LINK1_LFPS_MIN_GEN_U1_EXIT_SET;
2307                 writel(reg, &regs->dbg_link1);
2308         }
2309
2310         /*
2311          * By default some platforms has set protected access to memory.
2312          * This cause problem with cache, so driver restore non-secure
2313          * access to memory.
2314          */
2315         reg = readl(&regs->dma_axi_ctrl);
2316         reg |= DMA_AXI_CTRL_MARPROT(DMA_AXI_CTRL_NON_SECURE) |
2317                DMA_AXI_CTRL_MAWPROT(DMA_AXI_CTRL_NON_SECURE);
2318         writel(reg, &regs->dma_axi_ctrl);
2319
2320         /* enable generic interrupt*/
2321         writel(USB_IEN_INIT, &regs->usb_ien);
2322         writel(USB_CONF_CLK2OFFDS | USB_CONF_L1DS, &regs->usb_conf);
2323
2324         cdns3_configure_dmult(priv_dev, NULL);
2325
2326         cdns3_gadget_pullup(&priv_dev->gadget, 1);
2327 }
2328
2329 /**
2330  * cdns3_gadget_udc_start Gadget start
2331  * @gadget: gadget object
2332  * @driver: driver which operates on this gadget
2333  *
2334  * Returns 0 on success, error code elsewhere
2335  */
2336 static int cdns3_gadget_udc_start(struct usb_gadget *gadget,
2337                                   struct usb_gadget_driver *driver)
2338 {
2339         struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2340         unsigned long flags;
2341
2342         spin_lock_irqsave(&priv_dev->lock, flags);
2343         priv_dev->gadget_driver = driver;
2344         cdns3_gadget_config(priv_dev);
2345         spin_unlock_irqrestore(&priv_dev->lock, flags);
2346         return 0;
2347 }
2348
2349 /**
2350  * cdns3_gadget_udc_stop Stops gadget
2351  * @gadget: gadget object
2352  *
2353  * Returns 0
2354  */
2355 static int cdns3_gadget_udc_stop(struct usb_gadget *gadget)
2356 {
2357         struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2358         struct cdns3_endpoint *priv_ep;
2359         u32 bEndpointAddress;
2360         struct usb_ep *ep;
2361         int ret = 0;
2362         int val;
2363
2364         priv_dev->gadget_driver = NULL;
2365
2366         priv_dev->onchip_used_size = 0;
2367         priv_dev->out_mem_is_allocated = 0;
2368         priv_dev->gadget.speed = USB_SPEED_UNKNOWN;
2369
2370         list_for_each_entry(ep, &priv_dev->gadget.ep_list, ep_list) {
2371                 priv_ep = ep_to_cdns3_ep(ep);
2372                 bEndpointAddress = priv_ep->num | priv_ep->dir;
2373                 cdns3_select_ep(priv_dev, bEndpointAddress);
2374                 writel(EP_CMD_EPRST, &priv_dev->regs->ep_cmd);
2375                 readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
2376                                           !(val & EP_CMD_EPRST), 100);
2377         }
2378
2379         /* disable interrupt for device */
2380         writel(0, &priv_dev->regs->usb_ien);
2381         writel(USB_CONF_DEVDS, &priv_dev->regs->usb_conf);
2382
2383         return ret;
2384 }
2385
2386 static void cdns3_gadget_udc_set_speed(struct usb_gadget *gadget,
2387                                        enum usb_device_speed speed)
2388 {
2389         struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2390
2391         switch (speed) {
2392         case USB_SPEED_FULL:
2393                 writel(USB_CONF_SFORCE_FS, &priv_dev->regs->usb_conf);
2394                 writel(USB_CONF_USB3DIS, &priv_dev->regs->usb_conf);
2395                 break;
2396         case USB_SPEED_HIGH:
2397                 writel(USB_CONF_USB3DIS, &priv_dev->regs->usb_conf);
2398                 break;
2399         case USB_SPEED_SUPER:
2400                 break;
2401         default:
2402                 dev_err(cdns->dev, "invalid speed parameter %d\n",
2403                         speed);
2404         }
2405
2406         priv_dev->gadget.speed = speed;
2407 }
2408
2409 static const struct usb_gadget_ops cdns3_gadget_ops = {
2410         .get_frame = cdns3_gadget_get_frame,
2411         .wakeup = cdns3_gadget_wakeup,
2412         .set_selfpowered = cdns3_gadget_set_selfpowered,
2413         .pullup = cdns3_gadget_pullup,
2414         .udc_start = cdns3_gadget_udc_start,
2415         .udc_stop = cdns3_gadget_udc_stop,
2416         .match_ep = cdns3_gadget_match_ep,
2417         .udc_set_speed = cdns3_gadget_udc_set_speed,
2418 };
2419
2420 static void cdns3_free_all_eps(struct cdns3_device *priv_dev)
2421 {
2422         int i;
2423
2424         /* ep0 OUT point to ep0 IN. */
2425         priv_dev->eps[16] = NULL;
2426
2427         for (i = 0; i < CDNS3_ENDPOINTS_MAX_COUNT; i++)
2428                 if (priv_dev->eps[i]) {
2429                         cdns3_free_trb_pool(priv_dev->eps[i]);
2430                         devm_kfree(priv_dev->dev, priv_dev->eps[i]);
2431                 }
2432 }
2433
2434 /**
2435  * cdns3_init_eps Initializes software endpoints of gadget
2436  * @cdns3: extended gadget object
2437  *
2438  * Returns 0 on success, error code elsewhere
2439  */
2440 static int cdns3_init_eps(struct cdns3_device *priv_dev)
2441 {
2442         u32 ep_enabled_reg, iso_ep_reg;
2443         struct cdns3_endpoint *priv_ep;
2444         int ep_dir, ep_number;
2445         u32 ep_mask;
2446         int ret = 0;
2447         int i;
2448
2449         /* Read it from USB_CAP3 to USB_CAP5 */
2450         ep_enabled_reg = readl(&priv_dev->regs->usb_cap3);
2451         iso_ep_reg = readl(&priv_dev->regs->usb_cap4);
2452
2453         dev_dbg(priv_dev->dev, "Initializing non-zero endpoints\n");
2454
2455         for (i = 0; i < CDNS3_ENDPOINTS_MAX_COUNT; i++) {
2456                 ep_dir = i >> 4;        /* i div 16 */
2457                 ep_number = i & 0xF;    /* i % 16 */
2458                 ep_mask = BIT(i);
2459
2460                 if (!(ep_enabled_reg & ep_mask))
2461                         continue;
2462
2463                 if (ep_dir && !ep_number) {
2464                         priv_dev->eps[i] = priv_dev->eps[0];
2465                         continue;
2466                 }
2467
2468                 priv_ep = devm_kzalloc(priv_dev->dev, sizeof(*priv_ep),
2469                                        GFP_KERNEL);
2470                 if (!priv_ep) {
2471                         ret = -ENOMEM;
2472                         goto err;
2473                 }
2474
2475                 /* set parent of endpoint object */
2476                 priv_ep->cdns3_dev = priv_dev;
2477                 priv_dev->eps[i] = priv_ep;
2478                 priv_ep->num = ep_number;
2479                 priv_ep->dir = ep_dir ? USB_DIR_IN : USB_DIR_OUT;
2480
2481                 if (!ep_number) {
2482                         ret = cdns3_init_ep0(priv_dev, priv_ep);
2483                         if (ret) {
2484                                 dev_err(priv_dev->dev, "Failed to init ep0\n");
2485                                 goto err;
2486                         }
2487                 } else {
2488                         snprintf(priv_ep->name, sizeof(priv_ep->name), "ep%d%s",
2489                                  ep_number, !!ep_dir ? "in" : "out");
2490                         priv_ep->endpoint.name = priv_ep->name;
2491
2492                         usb_ep_set_maxpacket_limit(&priv_ep->endpoint,
2493                                                    CDNS3_EP_MAX_PACKET_LIMIT);
2494                         priv_ep->endpoint.max_streams = CDNS3_EP_MAX_STREAMS;
2495                         priv_ep->endpoint.ops = &cdns3_gadget_ep_ops;
2496                         if (ep_dir)
2497                                 priv_ep->endpoint.caps.dir_in = 1;
2498                         else
2499                                 priv_ep->endpoint.caps.dir_out = 1;
2500
2501                         if (iso_ep_reg & ep_mask)
2502                                 priv_ep->endpoint.caps.type_iso = 1;
2503
2504                         priv_ep->endpoint.caps.type_bulk = 1;
2505                         priv_ep->endpoint.caps.type_int = 1;
2506
2507                         list_add_tail(&priv_ep->endpoint.ep_list,
2508                                       &priv_dev->gadget.ep_list);
2509                 }
2510
2511                 priv_ep->flags = 0;
2512
2513                 dev_info(priv_dev->dev, "Initialized  %s support: %s %s\n",
2514                          priv_ep->name,
2515                          priv_ep->endpoint.caps.type_bulk ? "BULK, INT" : "",
2516                          priv_ep->endpoint.caps.type_iso ? "ISO" : "");
2517
2518                 INIT_LIST_HEAD(&priv_ep->pending_req_list);
2519                 INIT_LIST_HEAD(&priv_ep->deferred_req_list);
2520                 INIT_LIST_HEAD(&priv_ep->wa2_descmiss_req_list);
2521         }
2522
2523         return 0;
2524 err:
2525         cdns3_free_all_eps(priv_dev);
2526         return -ENOMEM;
2527 }
2528
2529 void cdns3_gadget_exit(struct cdns3 *cdns)
2530 {
2531         struct cdns3_device *priv_dev;
2532
2533         priv_dev = cdns->gadget_dev;
2534
2535         usb_del_gadget_udc(&priv_dev->gadget);
2536
2537         cdns3_free_all_eps(priv_dev);
2538
2539         while (!list_empty(&priv_dev->aligned_buf_list)) {
2540                 struct cdns3_aligned_buf *buf;
2541
2542                 buf = cdns3_next_align_buf(&priv_dev->aligned_buf_list);
2543                 dma_free_coherent(buf->buf);
2544
2545                 list_del(&buf->list);
2546                 kfree(buf);
2547         }
2548
2549         dma_free_coherent(priv_dev->setup_buf);
2550
2551         kfree(priv_dev->zlp_buf);
2552         kfree(priv_dev);
2553         cdns->gadget_dev = NULL;
2554         cdns3_drd_switch_gadget(cdns, 0);
2555 }
2556
2557 static int cdns3_gadget_start(struct cdns3 *cdns)
2558 {
2559         struct cdns3_device *priv_dev;
2560         u32 max_speed;
2561         int ret;
2562
2563         priv_dev = kzalloc(sizeof(*priv_dev), GFP_KERNEL);
2564         if (!priv_dev)
2565                 return -ENOMEM;
2566
2567         cdns->gadget_dev = priv_dev;
2568         priv_dev->sysdev = cdns->dev;
2569         priv_dev->dev = cdns->dev;
2570         priv_dev->regs = cdns->dev_regs;
2571
2572         dev_read_u32(priv_dev->dev, "cdns,on-chip-buff-size",
2573                      &priv_dev->onchip_buffers);
2574
2575         if (priv_dev->onchip_buffers <=  0) {
2576                 u32 reg = readl(&priv_dev->regs->usb_cap2);
2577
2578                 priv_dev->onchip_buffers = USB_CAP2_ACTUAL_MEM_SIZE(reg);
2579         }
2580
2581         if (!priv_dev->onchip_buffers)
2582                 priv_dev->onchip_buffers = 256;
2583
2584         max_speed = usb_get_maximum_speed(dev_ofnode(cdns->dev));
2585
2586         /* Check the maximum_speed parameter */
2587         switch (max_speed) {
2588         case USB_SPEED_FULL:
2589                 /* fall through */
2590         case USB_SPEED_HIGH:
2591                 /* fall through */
2592         case USB_SPEED_SUPER:
2593                 break;
2594         default:
2595                 dev_err(cdns->dev, "invalid maximum_speed parameter %d\n",
2596                         max_speed);
2597                 /* fall through */
2598         case USB_SPEED_UNKNOWN:
2599                 /* default to superspeed */
2600                 max_speed = USB_SPEED_SUPER;
2601                 break;
2602         }
2603
2604         /* fill gadget fields */
2605         priv_dev->gadget.max_speed = max_speed;
2606         priv_dev->gadget.speed = USB_SPEED_UNKNOWN;
2607         priv_dev->gadget.ops = &cdns3_gadget_ops;
2608         priv_dev->gadget.name = "cdns3-gadget";
2609 #ifndef __UBOOT__
2610         priv_dev->gadget.name = "usb-ss-gadget";
2611         priv_dev->gadget.sg_supported = 1;
2612         priv_dev->gadget.quirk_avoids_skb_reserve = 1;
2613 #endif
2614
2615         spin_lock_init(&priv_dev->lock);
2616         INIT_WORK(&priv_dev->pending_status_wq,
2617                   cdns3_pending_setup_status_handler);
2618
2619         /* initialize endpoint container */
2620         INIT_LIST_HEAD(&priv_dev->gadget.ep_list);
2621         INIT_LIST_HEAD(&priv_dev->aligned_buf_list);
2622
2623         ret = cdns3_init_eps(priv_dev);
2624         if (ret) {
2625                 dev_err(priv_dev->dev, "Failed to create endpoints\n");
2626                 goto err1;
2627         }
2628
2629         /* allocate memory for setup packet buffer */
2630         priv_dev->setup_buf =
2631                 dma_alloc_coherent(8, (unsigned long *)&priv_dev->setup_dma);
2632         if (!priv_dev->setup_buf) {
2633                 ret = -ENOMEM;
2634                 goto err2;
2635         }
2636
2637         priv_dev->dev_ver = readl(&priv_dev->regs->usb_cap6);
2638
2639         dev_dbg(priv_dev->dev, "Device Controller version: %08x\n",
2640                 readl(&priv_dev->regs->usb_cap6));
2641         dev_dbg(priv_dev->dev, "USB Capabilities:: %08x\n",
2642                 readl(&priv_dev->regs->usb_cap1));
2643         dev_dbg(priv_dev->dev, "On-Chip memory cnfiguration: %08x\n",
2644                 readl(&priv_dev->regs->usb_cap2));
2645
2646         priv_dev->dev_ver = GET_DEV_BASE_VERSION(priv_dev->dev_ver);
2647
2648         priv_dev->zlp_buf = kzalloc(CDNS3_EP_ZLP_BUF_SIZE, GFP_KERNEL);
2649         if (!priv_dev->zlp_buf) {
2650                 ret = -ENOMEM;
2651                 goto err3;
2652         }
2653
2654         /* add USB gadget device */
2655         ret = usb_add_gadget_udc((struct device *)priv_dev->dev,
2656                                  &priv_dev->gadget);
2657         if (ret < 0) {
2658                 dev_err(priv_dev->dev,
2659                         "Failed to register USB device controller\n");
2660                 goto err4;
2661         }
2662
2663         return 0;
2664 err4:
2665         kfree(priv_dev->zlp_buf);
2666 err3:
2667         dma_free_coherent(priv_dev->setup_buf);
2668 err2:
2669         cdns3_free_all_eps(priv_dev);
2670 err1:
2671         cdns->gadget_dev = NULL;
2672         return ret;
2673 }
2674
2675 static int __cdns3_gadget_init(struct cdns3 *cdns)
2676 {
2677         int ret = 0;
2678
2679         cdns3_drd_switch_gadget(cdns, 1);
2680
2681         ret = cdns3_gadget_start(cdns);
2682         if (ret)
2683                 return ret;
2684
2685         return 0;
2686 }
2687
2688 static int cdns3_gadget_suspend(struct cdns3 *cdns, bool do_wakeup)
2689 {
2690         struct cdns3_device *priv_dev = cdns->gadget_dev;
2691
2692         cdns3_disconnect_gadget(priv_dev);
2693
2694         priv_dev->gadget.speed = USB_SPEED_UNKNOWN;
2695         usb_gadget_set_state(&priv_dev->gadget, USB_STATE_NOTATTACHED);
2696         cdns3_hw_reset_eps_config(priv_dev);
2697
2698         /* disable interrupt for device */
2699         writel(0, &priv_dev->regs->usb_ien);
2700
2701         cdns3_gadget_pullup(&priv_dev->gadget, 0);
2702
2703         return 0;
2704 }
2705
2706 static int cdns3_gadget_resume(struct cdns3 *cdns, bool hibernated)
2707 {
2708         struct cdns3_device *priv_dev = cdns->gadget_dev;
2709
2710         if (!priv_dev->gadget_driver)
2711                 return 0;
2712
2713         cdns3_gadget_config(priv_dev);
2714
2715         return 0;
2716 }
2717
2718 /**
2719  * cdns3_gadget_init - initialize device structure
2720  *
2721  * cdns: cdns3 instance
2722  *
2723  * This function initializes the gadget.
2724  */
2725 int cdns3_gadget_init(struct cdns3 *cdns)
2726 {
2727         struct cdns3_role_driver *rdrv;
2728
2729         rdrv = devm_kzalloc(cdns->dev, sizeof(*rdrv), GFP_KERNEL);
2730         if (!rdrv)
2731                 return -ENOMEM;
2732
2733         rdrv->start     = __cdns3_gadget_init;
2734         rdrv->stop      = cdns3_gadget_exit;
2735         rdrv->suspend   = cdns3_gadget_suspend;
2736         rdrv->resume    = cdns3_gadget_resume;
2737         rdrv->state     = CDNS3_ROLE_STATE_INACTIVE;
2738         rdrv->name      = "gadget";
2739         cdns->roles[USB_ROLE_DEVICE] = rdrv;
2740
2741         return 0;
2742 }
2743
2744 /**
2745  * cdns3_gadget_uboot_handle_interrupt - handle cdns3 gadget interrupt
2746  * @cdns: pointer to struct cdns3
2747  *
2748  * Handles ep0 and gadget interrupt
2749  */
2750 static void cdns3_gadget_uboot_handle_interrupt(struct cdns3 *cdns)
2751 {
2752         int ret = cdns3_device_irq_handler(0, cdns);
2753
2754         if (ret == IRQ_WAKE_THREAD)
2755                 cdns3_device_thread_irq_handler(0, cdns);
2756 }
2757
2758 int dm_usb_gadget_handle_interrupts(struct udevice *dev)
2759 {
2760         struct cdns3 *cdns = dev_get_priv(dev);
2761
2762         cdns3_gadget_uboot_handle_interrupt(cdns);
2763
2764         return 0;
2765 }