drivers/staging/octeon-usb/octeon-hcd.c

   1 /*
   2  * This file is subject to the terms and conditions of the GNU General Public
   3  * License.  See the file "COPYING" in the main directory of this archive
   4  * for more details.
   5  *
   6  * Copyright (C) 2008 Cavium Networks
   7  *
   8  * Some parts of the code were originally released under BSD license:
   9  *
  10  * Copyright (c) 2003-2010 Cavium Networks (support@cavium.com). All rights
  11  * reserved.
  12  *
  13  * Redistribution and use in source and binary forms, with or without
  14  * modification, are permitted provided that the following conditions are
  15  * met:
  16  *
  17  *   * Redistributions of source code must retain the above copyright
  18  *     notice, this list of conditions and the following disclaimer.
  19  *
  20  *   * Redistributions in binary form must reproduce the above
  21  *     copyright notice, this list of conditions and the following
  22  *     disclaimer in the documentation and/or other materials provided
  23  *     with the distribution.
  24  *
  25  *   * Neither the name of Cavium Networks nor the names of
  26  *     its contributors may be used to endorse or promote products
  27  *     derived from this software without specific prior written
  28  *     permission.
  29  *
  30  * This Software, including technical data, may be subject to U.S. export
  31  * control laws, including the U.S. Export Administration Act and its associated
  32  * regulations, and may be subject to export or import regulations in other
  33  * countries.
  34  *
  35  * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS"
  36  * AND WITH ALL FAULTS AND CAVIUM NETWORKS MAKES NO PROMISES, REPRESENTATIONS OR
  37  * WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH RESPECT TO
  38  * THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY REPRESENTATION
  39  * OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT DEFECTS, AND CAVIUM
  40  * SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) WARRANTIES OF TITLE,
  41  * MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A PARTICULAR PURPOSE, LACK OF
  42  * VIRUSES, ACCURACY OR COMPLETENESS, QUIET ENJOYMENT, QUIET POSSESSION OR
  43  * CORRESPONDENCE TO DESCRIPTION. THE ENTIRE RISK ARISING OUT OF USE OR
  44  * PERFORMANCE OF THE SOFTWARE LIES WITH YOU.
  45  */
  46 #include <linux/kernel.h>
  47 #include <linux/module.h>
  48 #include <linux/init.h>
  49 #include <linux/pci.h>
  50 #include <linux/prefetch.h>
  51 #include <linux/interrupt.h>
  52 #include <linux/platform_device.h>
  53 #include <linux/usb.h>
  54
  55 #include <linux/time.h>
  56 #include <linux/delay.h>
  57
  58 #include <asm/octeon/cvmx.h>
  59 #include <asm/octeon/cvmx-iob-defs.h>
  60
  61 #include <linux/usb/hcd.h>
  62
  63 #include <linux/err.h>
  64
  65 #include <asm/octeon/octeon.h>
  66 #include <asm/octeon/cvmx-helper.h>
  67 #include <asm/octeon/cvmx-sysinfo.h>
  68 #include <asm/octeon/cvmx-helper-board.h>
  69
  70 #include "octeon-hcd.h"
  71
  72 /**
  73  * enum cvmx_usb_speed - the possible USB device speeds
  74  *
  75  * @CVMX_USB_SPEED_HIGH: Device is operation at 480Mbps
  76  * @CVMX_USB_SPEED_FULL: Device is operation at 12Mbps
  77  * @CVMX_USB_SPEED_LOW:  Device is operation at 1.5Mbps
  78  */
  79 enum cvmx_usb_speed {
  80         CVMX_USB_SPEED_HIGH = 0,
  81         CVMX_USB_SPEED_FULL = 1,
  82         CVMX_USB_SPEED_LOW = 2,
  83 };
  84
  85 /**
  86  * enum cvmx_usb_transfer - the possible USB transfer types
  87  *
  88  * @CVMX_USB_TRANSFER_CONTROL:     USB transfer type control for hub and status
  89  *                                 transfers
  90  * @CVMX_USB_TRANSFER_ISOCHRONOUS: USB transfer type isochronous for low
  91  *                                 priority periodic transfers
  92  * @CVMX_USB_TRANSFER_BULK:        USB transfer type bulk for large low priority
  93  *                                 transfers
  94  * @CVMX_USB_TRANSFER_INTERRUPT:   USB transfer type interrupt for high priority
  95  *                                 periodic transfers
  96  */
  97 enum cvmx_usb_transfer {
  98         CVMX_USB_TRANSFER_CONTROL = 0,
  99         CVMX_USB_TRANSFER_ISOCHRONOUS = 1,
 100         CVMX_USB_TRANSFER_BULK = 2,
 101         CVMX_USB_TRANSFER_INTERRUPT = 3,
 102 };
 103
 104 /**
 105  * enum cvmx_usb_direction - the transfer directions
 106  *
 107  * @CVMX_USB_DIRECTION_OUT: Data is transferring from Octeon to the device/host
 108  * @CVMX_USB_DIRECTION_IN:  Data is transferring from the device/host to Octeon
 109  */
 110 enum cvmx_usb_direction {
 111         CVMX_USB_DIRECTION_OUT,
 112         CVMX_USB_DIRECTION_IN,
 113 };
 114
 115 /**
 116  * enum cvmx_usb_complete - possible callback function status codes
 117  *
 118  * @CVMX_USB_COMPLETE_SUCCESS:    The transaction / operation finished without
 119  *                                any errors
 120  * @CVMX_USB_COMPLETE_SHORT:      FIXME: This is currently not implemented
 121  * @CVMX_USB_COMPLETE_CANCEL:     The transaction was canceled while in flight
 122  *                                by a user call to cvmx_usb_cancel
 123  * @CVMX_USB_COMPLETE_ERROR:      The transaction aborted with an unexpected
 124  *                                error status
 125  * @CVMX_USB_COMPLETE_STALL:      The transaction received a USB STALL response
 126  *                                from the device
 127  * @CVMX_USB_COMPLETE_XACTERR:    The transaction failed with an error from the
 128  *                                device even after a number of retries
 129  * @CVMX_USB_COMPLETE_DATATGLERR: The transaction failed with a data toggle
 130  *                                error even after a number of retries
 131  * @CVMX_USB_COMPLETE_BABBLEERR:  The transaction failed with a babble error
 132  * @CVMX_USB_COMPLETE_FRAMEERR:   The transaction failed with a frame error
 133  *                                even after a number of retries
 134  */
 135 enum cvmx_usb_complete {
 136         CVMX_USB_COMPLETE_SUCCESS,
 137         CVMX_USB_COMPLETE_SHORT,
 138         CVMX_USB_COMPLETE_CANCEL,
 139         CVMX_USB_COMPLETE_ERROR,
 140         CVMX_USB_COMPLETE_STALL,
 141         CVMX_USB_COMPLETE_XACTERR,
 142         CVMX_USB_COMPLETE_DATATGLERR,
 143         CVMX_USB_COMPLETE_BABBLEERR,
 144         CVMX_USB_COMPLETE_FRAMEERR,
 145 };
 146
 147 /**
 148  * struct cvmx_usb_port_status - the USB port status information
 149  *
 150  * @port_enabled:       1 = Usb port is enabled, 0 = disabled
 151  * @port_over_current:  1 = Over current detected, 0 = Over current not
 152  *                      detected. Octeon doesn't support over current detection.
 153  * @port_powered:       1 = Port power is being supplied to the device, 0 =
 154  *                      power is off. Octeon doesn't support turning port power
 155  *                      off.
 156  * @port_speed:         Current port speed.
 157  * @connected:          1 = A device is connected to the port, 0 = No device is
 158  *                      connected.
 159  * @connect_change:     1 = Device connected state changed since the last set
 160  *                      status call.
 161  */
 162 struct cvmx_usb_port_status {
 163         uint32_t reserved               : 25;
 164         uint32_t port_enabled           : 1;
 165         uint32_t port_over_current      : 1;
 166         uint32_t port_powered           : 1;
 167         enum cvmx_usb_speed port_speed  : 2;
 168         uint32_t connected              : 1;
 169         uint32_t connect_change         : 1;
 170 };
 171
 172 /**
 173  * union cvmx_usb_control_header - the structure of a Control packet header
 174  *
 175  * @s.request_type:     Bit 7 tells the direction: 1=IN, 0=OUT
 176  * @s.request           The standard usb request to make
 177  * @s.value             Value parameter for the request in little endian format
 178  * @s.index             Index for the request in little endian format
 179  * @s.length            Length of the data associated with this request in
 180  *                      little endian format
 181  */
 182 union cvmx_usb_control_header {
 183         uint64_t u64;
 184         struct {
 185                 uint64_t request_type   : 8;
 186                 uint64_t request        : 8;
 187                 uint64_t value          : 16;
 188                 uint64_t index          : 16;
 189                 uint64_t length         : 16;
 190         } s;
 191 };
 192
 193 /**
 194  * struct cvmx_usb_iso_packet - descriptor for Isochronous packets
 195  *
 196  * @offset:     This is the offset in bytes into the main buffer where this data
 197  *              is stored.
 198  * @length:     This is the length in bytes of the data.
 199  * @status:     This is the status of this individual packet transfer.
 200  */
 201 struct cvmx_usb_iso_packet {
 202         int offset;
 203         int length;
 204         enum cvmx_usb_complete status;
 205 };
 206
 207 /**
 208  * enum cvmx_usb_initialize_flags - flags used by the initialization function
 209  *
 210  * @CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_XI:    The USB port uses a 12MHz crystal
 211  *                                            as clock source at USB_XO and
 212  *                                            USB_XI.
 213  * @CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_GND:   The USB port uses 12/24/48MHz 2.5V
 214  *                                            board clock source at USB_XO.
 215  *                                            USB_XI should be tied to GND.
 216  * @CVMX_USB_INITIALIZE_FLAGS_CLOCK_MHZ_MASK: Mask for clock speed field
 217  * @CVMX_USB_INITIALIZE_FLAGS_CLOCK_12MHZ:    Speed of reference clock or
 218  *                                            crystal
 219  * @CVMX_USB_INITIALIZE_FLAGS_CLOCK_24MHZ:    Speed of reference clock
 220  * @CVMX_USB_INITIALIZE_FLAGS_CLOCK_48MHZ:    Speed of reference clock
 221  * @CVMX_USB_INITIALIZE_FLAGS_NO_DMA:         Disable DMA and used polled IO for
 222  *                                            data transfer use for the USB
 223  */
 224 enum cvmx_usb_initialize_flags {
 225         CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_XI           = 1 << 0,
 226         CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_GND          = 1 << 1,
 227         CVMX_USB_INITIALIZE_FLAGS_CLOCK_MHZ_MASK        = 3 << 3,
 228         CVMX_USB_INITIALIZE_FLAGS_CLOCK_12MHZ           = 1 << 3,
 229         CVMX_USB_INITIALIZE_FLAGS_CLOCK_24MHZ           = 2 << 3,
 230         CVMX_USB_INITIALIZE_FLAGS_CLOCK_48MHZ           = 3 << 3,
 231         /* Bits 3-4 used to encode the clock frequency */
 232         CVMX_USB_INITIALIZE_FLAGS_NO_DMA                = 1 << 5,
 233 };
 234
 235 /**
 236  * enum cvmx_usb_pipe_flags - internal flags for a pipe.
 237  *
 238  * @__CVMX_USB_PIPE_FLAGS_SCHEDULED: Used internally to determine if a pipe is
 239  *                                   actively using hardware. Do not use.
 240  * @__CVMX_USB_PIPE_FLAGS_NEED_PING: Used internally to determine if a high
 241  *                                   speed pipe is in the ping state. Do not
 242  *                                   use.
 243  */
 244 enum cvmx_usb_pipe_flags {
 245         __CVMX_USB_PIPE_FLAGS_SCHEDULED = 1 << 17,
 246         __CVMX_USB_PIPE_FLAGS_NEED_PING = 1 << 18,
 247 };
 248
 249 /* Maximum number of times to retry failed transactions */
 250 #define MAX_RETRIES             3
 251
 252 /* Maximum number of hardware channels supported by the USB block */
 253 #define MAX_CHANNELS            8
 254
 255 /* The highest valid USB device address */
 256 #define MAX_USB_ADDRESS         127
 257
 258 /* The highest valid USB endpoint number */
 259 #define MAX_USB_ENDPOINT        15
 260
 261 /* The highest valid port number on a hub */
 262 #define MAX_USB_HUB_PORT        15
 263
 264 /*
 265  * The low level hardware can transfer a maximum of this number of bytes in each
 266  * transfer. The field is 19 bits wide
 267  */
 268 #define MAX_TRANSFER_BYTES      ((1<<19)-1)
 269
 270 /*
 271  * The low level hardware can transfer a maximum of this number of packets in
 272  * each transfer. The field is 10 bits wide
 273  */
 274 #define MAX_TRANSFER_PACKETS    ((1<<10)-1)
 275
 276 /**
 277  * Logical transactions may take numerous low level
 278  * transactions, especially when splits are concerned. This
 279  * enum represents all of the possible stages a transaction can
 280  * be in. Note that split completes are always even. This is so
 281  * the NAK handler can backup to the previous low level
 282  * transaction with a simple clearing of bit 0.
 283  */
 284 enum cvmx_usb_stage {
 285         CVMX_USB_STAGE_NON_CONTROL,
 286         CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE,
 287         CVMX_USB_STAGE_SETUP,
 288         CVMX_USB_STAGE_SETUP_SPLIT_COMPLETE,
 289         CVMX_USB_STAGE_DATA,
 290         CVMX_USB_STAGE_DATA_SPLIT_COMPLETE,
 291         CVMX_USB_STAGE_STATUS,
 292         CVMX_USB_STAGE_STATUS_SPLIT_COMPLETE,
 293 };
 294
 295 /**
 296  * struct cvmx_usb_transaction - describes each pending USB transaction
 297  *                               regardless of type. These are linked together
 298  *                               to form a list of pending requests for a pipe.
 299  *
 300  * @node:               List node for transactions in the pipe.
 301  * @type:               Type of transaction, duplicated of the pipe.
 302  * @flags:              State flags for this transaction.
 303  * @buffer:             User's physical buffer address to read/write.
 304  * @buffer_length:      Size of the user's buffer in bytes.
 305  * @control_header:     For control transactions, physical address of the 8
 306  *                      byte standard header.
 307  * @iso_start_frame:    For ISO transactions, the starting frame number.
 308  * @iso_number_packets: For ISO transactions, the number of packets in the
 309  *                      request.
 310  * @iso_packets:        For ISO transactions, the sub packets in the request.
 311  * @actual_bytes:       Actual bytes transfer for this transaction.
 312  * @stage:              For control transactions, the current stage.
 313  * @urb:                URB.
 314  */
 315 struct cvmx_usb_transaction {
 316         struct list_head node;
 317         enum cvmx_usb_transfer type;
 318         uint64_t buffer;
 319         int buffer_length;
 320         uint64_t control_header;
 321         int iso_start_frame;
 322         int iso_number_packets;
 323         struct cvmx_usb_iso_packet *iso_packets;
 324         int xfersize;
 325         int pktcnt;
 326         int retries;
 327         int actual_bytes;
 328         enum cvmx_usb_stage stage;
 329         struct urb *urb;
 330 };
 331
 332 /**
 333  * struct cvmx_usb_pipe - a pipe represents a virtual connection between Octeon
 334  *                        and some USB device. It contains a list of pending
 335  *                        request to the device.
 336  *
 337  * @node:               List node for pipe list
 338  * @next:               Pipe after this one in the list
 339  * @transactions:       List of pending transactions
 340  * @interval:           For periodic pipes, the interval between packets in
 341  *                      frames
 342  * @next_tx_frame:      The next frame this pipe is allowed to transmit on
 343  * @flags:              State flags for this pipe
 344  * @device_speed:       Speed of device connected to this pipe
 345  * @transfer_type:      Type of transaction supported by this pipe
 346  * @transfer_dir:       IN or OUT. Ignored for Control
 347  * @multi_count:        Max packet in a row for the device
 348  * @max_packet:         The device's maximum packet size in bytes
 349  * @device_addr:        USB device address at other end of pipe
 350  * @endpoint_num:       USB endpoint number at other end of pipe
 351  * @hub_device_addr:    Hub address this device is connected to
 352  * @hub_port:           Hub port this device is connected to
 353  * @pid_toggle:         This toggles between 0/1 on every packet send to track
 354  *                      the data pid needed
 355  * @channel:            Hardware DMA channel for this pipe
 356  * @split_sc_frame:     The low order bits of the frame number the split
 357  *                      complete should be sent on
 358  */
 359 struct cvmx_usb_pipe {
 360         struct list_head node;
 361         struct list_head transactions;
 362         uint64_t interval;
 363         uint64_t next_tx_frame;
 364         enum cvmx_usb_pipe_flags flags;
 365         enum cvmx_usb_speed device_speed;
 366         enum cvmx_usb_transfer transfer_type;
 367         enum cvmx_usb_direction transfer_dir;
 368         int multi_count;
 369         uint16_t max_packet;
 370         uint8_t device_addr;
 371         uint8_t endpoint_num;
 372         uint8_t hub_device_addr;
 373         uint8_t hub_port;
 374         uint8_t pid_toggle;
 375         uint8_t channel;
 376         int8_t split_sc_frame;
 377 };
 378
 379 struct cvmx_usb_tx_fifo {
 380         struct {
 381                 int channel;
 382                 int size;
 383                 uint64_t address;
 384         } entry[MAX_CHANNELS+1];
 385         int head;
 386         int tail;
 387 };
 388
 389 /**
 390  * struct cvmx_usb_state - the state of the USB block
 391  *
 392  * init_flags:             Flags passed to initialize.
 393  * index:                  Which USB block this is for.
 394  * idle_hardware_channels: Bit set for every idle hardware channel.
 395  * usbcx_hprt:             Stored port status so we don't need to read a CSR to
 396  *                         determine splits.
 397  * pipe_for_channel:       Map channels to pipes.
 398  * pipe:                   Storage for pipes.
 399  * indent:                 Used by debug output to indent functions.
 400  * port_status:            Last port status used for change notification.
 401  * idle_pipes:             List of open pipes that have no transactions.
 402  * active_pipes:           Active pipes indexed by transfer type.
 403  * frame_number:           Increments every SOF interrupt for time keeping.
 404  * active_split:           Points to the current active split, or NULL.
 405  */
 406 struct cvmx_usb_state {
 407         int init_flags;
 408         int index;
 409         int idle_hardware_channels;
 410         union cvmx_usbcx_hprt usbcx_hprt;
 411         struct cvmx_usb_pipe *pipe_for_channel[MAX_CHANNELS];
 412         int indent;
 413         struct cvmx_usb_port_status port_status;
 414         struct list_head idle_pipes;
 415         struct list_head active_pipes[4];
 416         uint64_t frame_number;
 417         struct cvmx_usb_transaction *active_split;
 418         struct cvmx_usb_tx_fifo periodic;
 419         struct cvmx_usb_tx_fifo nonperiodic;
 420 };
 421
 422 struct octeon_hcd {
 423         spinlock_t lock;
 424         struct cvmx_usb_state usb;
 425 };
 426
 427 /* This macro spins on a field waiting for it to reach a value */
 428 #define CVMX_WAIT_FOR_FIELD32(address, type, field, op, value, timeout_usec)\
 429         ({int result;                                                       \
 430         do {                                                                \
 431                 uint64_t done = cvmx_get_cycle() + (uint64_t)timeout_usec * \
 432                         octeon_get_clock_rate() / 1000000;                  \
 433                 type c;                                                     \
 434                 while (1) {                                                 \
 435                         c.u32 = __cvmx_usb_read_csr32(usb, address);        \
 436                         if (c.s.field op (value)) {                         \
 437                                 result = 0;                                 \
 438                                 break;                                      \
 439                         } else if (cvmx_get_cycle() > done) {               \
 440                                 result = -1;                                \
 441                                 break;                                      \
 442                         } else                                              \
 443                                 cvmx_wait(100);                             \
 444                 }                                                           \
 445         } while (0);                                                        \
 446         result; })
 447
 448 /*
 449  * This macro logically sets a single field in a CSR. It does the sequence
 450  * read, modify, and write
 451  */
 452 #define USB_SET_FIELD32(address, type, field, value)            \
 453         do {                                                    \
 454                 type c;                                         \
 455                 c.u32 = __cvmx_usb_read_csr32(usb, address);    \
 456                 c.s.field = value;                              \
 457                 __cvmx_usb_write_csr32(usb, address, c.u32);    \
 458         } while (0)
 459
 460 /* Returns the IO address to push/pop stuff data from the FIFOs */
 461 #define USB_FIFO_ADDRESS(channel, usb_index) \
 462         (CVMX_USBCX_GOTGCTL(usb_index) + ((channel)+1)*0x1000)
 463
 464 /**
 465  * struct octeon_temp_buffer - a bounce buffer for USB transfers
 466  * @temp_buffer: the newly allocated temporary buffer (including meta-data)
 467  * @orig_buffer: the original buffer passed by the USB stack
 468  * @data:        the newly allocated temporary buffer (excluding meta-data)
 469  *
 470  * Both the DMA engine and FIFO mode will always transfer full 32-bit words. If
 471  * the buffer is too short, we need to allocate a temporary one, and this struct
 472  * represents it.
 473  */
 474 struct octeon_temp_buffer {
 475         void *temp_buffer;
 476         void *orig_buffer;
 477         u8 data[0];
 478 };
 479
 480 /**
 481  * octeon_alloc_temp_buffer - allocate a temporary buffer for USB transfer
 482  *                            (if needed)
 483  * @urb:        URB.
 484  * @mem_flags:  Memory allocation flags.
 485  *
 486  * This function allocates a temporary bounce buffer whenever it's needed
 487  * due to HW limitations.
 488  */
 489 static int octeon_alloc_temp_buffer(struct urb *urb, gfp_t mem_flags)
 490 {
 491         struct octeon_temp_buffer *temp;
 492
 493         if (urb->num_sgs || urb->sg ||
 494             (urb->transfer_flags & URB_NO_TRANSFER_DMA_MAP) ||
 495             !(urb->transfer_buffer_length % sizeof(u32)))
 496                 return 0;
 497
 498         temp = kmalloc(ALIGN(urb->transfer_buffer_length, sizeof(u32)) +
 499                        sizeof(*temp), mem_flags);
 500         if (!temp)
 501                 return -ENOMEM;
 502
 503         temp->temp_buffer = temp;
 504         temp->orig_buffer = urb->transfer_buffer;
 505         if (usb_urb_dir_out(urb))
 506                 memcpy(temp->data, urb->transfer_buffer,
 507                        urb->transfer_buffer_length);
 508         urb->transfer_buffer = temp->data;
 509         urb->transfer_flags |= URB_ALIGNED_TEMP_BUFFER;
 510
 511         return 0;
 512 }
 513
 514 /**
 515  * octeon_free_temp_buffer - free a temporary buffer used by USB transfers.
 516  * @urb: URB.
 517  *
 518  * Frees a buffer allocated by octeon_alloc_temp_buffer().
 519  */
 520 static void octeon_free_temp_buffer(struct urb *urb)
 521 {
 522         struct octeon_temp_buffer *temp;
 523
 524         if (!(urb->transfer_flags & URB_ALIGNED_TEMP_BUFFER))
 525                 return;
 526
 527         temp = container_of(urb->transfer_buffer, struct octeon_temp_buffer,
 528                             data);
 529         if (usb_urb_dir_in(urb))
 530                 memcpy(temp->orig_buffer, urb->transfer_buffer,
 531                        urb->actual_length);
 532         urb->transfer_buffer = temp->orig_buffer;
 533         urb->transfer_flags &= ~URB_ALIGNED_TEMP_BUFFER;
 534         kfree(temp->temp_buffer);
 535 }
 536
 537 /**
 538  * octeon_map_urb_for_dma - Octeon-specific map_urb_for_dma().
 539  * @hcd:        USB HCD structure.
 540  * @urb:        URB.
 541  * @mem_flags:  Memory allocation flags.
 542  */
 543 static int octeon_map_urb_for_dma(struct usb_hcd *hcd, struct urb *urb,
 544                                   gfp_t mem_flags)
 545 {
 546         int ret;
 547
 548         ret = octeon_alloc_temp_buffer(urb, mem_flags);
 549         if (ret)
 550                 return ret;
 551
 552         ret = usb_hcd_map_urb_for_dma(hcd, urb, mem_flags);
 553         if (ret)
 554                 octeon_free_temp_buffer(urb);
 555
 556         return ret;
 557 }
 558
 559 /**
 560  * octeon_unmap_urb_for_dma - Octeon-specific unmap_urb_for_dma()
 561  * @hcd:        USB HCD structure.
 562  * @urb:        URB.
 563  */
 564 static void octeon_unmap_urb_for_dma(struct usb_hcd *hcd, struct urb *urb)
 565 {
 566         usb_hcd_unmap_urb_for_dma(hcd, urb);
 567         octeon_free_temp_buffer(urb);
 568 }
 569
 570 /**
 571  * Read a USB 32bit CSR. It performs the necessary address swizzle
 572  * for 32bit CSRs and logs the value in a readable format if
 573  * debugging is on.
 574  *
 575  * @usb:     USB block this access is for
 576  * @address: 64bit address to read
 577  *
 578  * Returns: Result of the read
 579  */
 580 static inline uint32_t __cvmx_usb_read_csr32(struct cvmx_usb_state *usb,
 581                                              uint64_t address)
 582 {
 583         uint32_t result = cvmx_read64_uint32(address ^ 4);
 584         return result;
 585 }
 586
 587
 588 /**
 589  * Write a USB 32bit CSR. It performs the necessary address
 590  * swizzle for 32bit CSRs and logs the value in a readable format
 591  * if debugging is on.
 592  *
 593  * @usb:     USB block this access is for
 594  * @address: 64bit address to write
 595  * @value:   Value to write
 596  */
 597 static inline void __cvmx_usb_write_csr32(struct cvmx_usb_state *usb,
 598                                           uint64_t address, uint32_t value)
 599 {
 600         cvmx_write64_uint32(address ^ 4, value);
 601         cvmx_read64_uint64(CVMX_USBNX_DMA0_INB_CHN0(usb->index));
 602 }
 603
 604
 605 /**
 606  * Read a USB 64bit CSR. It logs the value in a readable format if
 607  * debugging is on.
 608  *
 609  * @usb:     USB block this access is for
 610  * @address: 64bit address to read
 611  *
 612  * Returns: Result of the read
 613  */
 614 static inline uint64_t __cvmx_usb_read_csr64(struct cvmx_usb_state *usb,
 615                                              uint64_t address)
 616 {
 617         uint64_t result = cvmx_read64_uint64(address);
 618         return result;
 619 }
 620
 621
 622 /**
 623  * Write a USB 64bit CSR. It logs the value in a readable format
 624  * if debugging is on.
 625  *
 626  * @usb:     USB block this access is for
 627  * @address: 64bit address to write
 628  * @value:   Value to write
 629  */
 630 static inline void __cvmx_usb_write_csr64(struct cvmx_usb_state *usb,
 631                                           uint64_t address, uint64_t value)
 632 {
 633         cvmx_write64_uint64(address, value);
 634 }
 635
 636 /**
 637  * Return non zero if this pipe connects to a non HIGH speed
 638  * device through a high speed hub.
 639  *
 640  * @usb:    USB block this access is for
 641  * @pipe:   Pipe to check
 642  *
 643  * Returns: Non zero if we need to do split transactions
 644  */
 645 static inline int __cvmx_usb_pipe_needs_split(struct cvmx_usb_state *usb,
 646                                               struct cvmx_usb_pipe *pipe)
 647 {
 648         return pipe->device_speed != CVMX_USB_SPEED_HIGH &&
 649                usb->usbcx_hprt.s.prtspd == CVMX_USB_SPEED_HIGH;
 650 }
 651
 652
 653 /**
 654  * Trivial utility function to return the correct PID for a pipe
 655  *
 656  * @pipe:   pipe to check
 657  *
 658  * Returns: PID for pipe
 659  */
 660 static inline int __cvmx_usb_get_data_pid(struct cvmx_usb_pipe *pipe)
 661 {
 662         if (pipe->pid_toggle)
 663                 return 2; /* Data1 */
 664         else
 665                 return 0; /* Data0 */
 666 }
 667
 668 /**
 669  * Initialize a USB port for use. This must be called before any
 670  * other access to the Octeon USB port is made. The port starts
 671  * off in the disabled state.
 672  *
 673  * @usb:         Pointer to an empty struct cvmx_usb_state
 674  *               that will be populated by the initialize call.
 675  *               This structure is then passed to all other USB
 676  *               functions.
 677  * @usb_port_number:
 678  *               Which Octeon USB port to initialize.
 679  *
 680  * Returns: 0 or a negative error code.
 681  */
 682 static int cvmx_usb_initialize(struct cvmx_usb_state *usb,
 683                                int usb_port_number,
 684                                enum cvmx_usb_initialize_flags flags)
 685 {
 686         union cvmx_usbnx_clk_ctl usbn_clk_ctl;
 687         union cvmx_usbnx_usbp_ctl_status usbn_usbp_ctl_status;
 688         int i;
 689
 690         /* At first allow 0-1 for the usb port number */
 691         if ((usb_port_number < 0) || (usb_port_number > 1))
 692                 return -EINVAL;
 693
 694         memset(usb, 0, sizeof(*usb));
 695         usb->init_flags = flags;
 696
 697         /* Initialize the USB state structure */
 698         usb->index = usb_port_number;
 699         INIT_LIST_HEAD(&usb->idle_pipes);
 700         for (i = 0; i < ARRAY_SIZE(usb->active_pipes); i++)
 701                 INIT_LIST_HEAD(&usb->active_pipes[i]);
 702
 703         /*
 704          * Power On Reset and PHY Initialization
 705          *
 706          * 1. Wait for DCOK to assert (nothing to do)
 707          *
 708          * 2a. Write USBN0/1_CLK_CTL[POR] = 1 and
 709          *     USBN0/1_CLK_CTL[HRST,PRST,HCLK_RST] = 0
 710          */
 711         usbn_clk_ctl.u64 =
 712                 __cvmx_usb_read_csr64(usb, CVMX_USBNX_CLK_CTL(usb->index));
 713         usbn_clk_ctl.s.por = 1;
 714         usbn_clk_ctl.s.hrst = 0;
 715         usbn_clk_ctl.s.prst = 0;
 716         usbn_clk_ctl.s.hclk_rst = 0;
 717         usbn_clk_ctl.s.enable = 0;
 718         /*
 719          * 2b. Select the USB reference clock/crystal parameters by writing
 720          *     appropriate values to USBN0/1_CLK_CTL[P_C_SEL, P_RTYPE, P_COM_ON]
 721          */
 722         if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_GND) {
 723                 /*
 724                  * The USB port uses 12/24/48MHz 2.5V board clock
 725                  * source at USB_XO. USB_XI should be tied to GND.
 726                  * Most Octeon evaluation boards require this setting
 727                  */
 728                 if (OCTEON_IS_MODEL(OCTEON_CN3XXX) ||
 729                     OCTEON_IS_MODEL(OCTEON_CN56XX) ||
 730                     OCTEON_IS_MODEL(OCTEON_CN50XX))
 731                         /* From CN56XX,CN50XX,CN31XX,CN30XX manuals */
 732                         usbn_clk_ctl.s.p_rtype = 2; /* p_rclk=1 & p_xenbn=0 */
 733                 else
 734                         /* From CN52XX manual */
 735                         usbn_clk_ctl.s.p_rtype = 1;
 736
 737                 switch (flags & CVMX_USB_INITIALIZE_FLAGS_CLOCK_MHZ_MASK) {
 738                 case CVMX_USB_INITIALIZE_FLAGS_CLOCK_12MHZ:
 739                         usbn_clk_ctl.s.p_c_sel = 0;
 740                         break;
 741                 case CVMX_USB_INITIALIZE_FLAGS_CLOCK_24MHZ:
 742                         usbn_clk_ctl.s.p_c_sel = 1;
 743                         break;
 744                 case CVMX_USB_INITIALIZE_FLAGS_CLOCK_48MHZ:
 745                         usbn_clk_ctl.s.p_c_sel = 2;
 746                         break;
 747                 }
 748         } else {
 749                 /*
 750                  * The USB port uses a 12MHz crystal as clock source
 751                  * at USB_XO and USB_XI
 752                  */
 753                 if (OCTEON_IS_MODEL(OCTEON_CN3XXX))
 754                         /* From CN31XX,CN30XX manual */
 755                         usbn_clk_ctl.s.p_rtype = 3; /* p_rclk=1 & p_xenbn=1 */
 756                 else
 757                         /* From CN56XX,CN52XX,CN50XX manuals. */
 758                         usbn_clk_ctl.s.p_rtype = 0;
 759
 760                 usbn_clk_ctl.s.p_c_sel = 0;
 761         }
 762         /*
 763          * 2c. Select the HCLK via writing USBN0/1_CLK_CTL[DIVIDE, DIVIDE2] and
 764          *     setting USBN0/1_CLK_CTL[ENABLE] = 1. Divide the core clock down
 765          *     such that USB is as close as possible to 125Mhz
 766          */
 767         {
 768                 int divisor = (octeon_get_clock_rate()+125000000-1)/125000000;
 769                 /* Lower than 4 doesn't seem to work properly */
 770                 if (divisor < 4)
 771                         divisor = 4;
 772                 usbn_clk_ctl.s.divide = divisor;
 773                 usbn_clk_ctl.s.divide2 = 0;
 774         }
 775         __cvmx_usb_write_csr64(usb, CVMX_USBNX_CLK_CTL(usb->index),
 776                                usbn_clk_ctl.u64);
 777         /* 2d. Write USBN0/1_CLK_CTL[HCLK_RST] = 1 */
 778         usbn_clk_ctl.s.hclk_rst = 1;
 779         __cvmx_usb_write_csr64(usb, CVMX_USBNX_CLK_CTL(usb->index),
 780                                usbn_clk_ctl.u64);
 781         /* 2e.  Wait 64 core-clock cycles for HCLK to stabilize */
 782         cvmx_wait(64);
 783         /*
 784          * 3. Program the power-on reset field in the USBN clock-control
 785          *    register:
 786          *    USBN_CLK_CTL[POR] = 0
 787          */
 788         usbn_clk_ctl.s.por = 0;
 789         __cvmx_usb_write_csr64(usb, CVMX_USBNX_CLK_CTL(usb->index),
 790                                usbn_clk_ctl.u64);
 791         /* 4. Wait 1 ms for PHY clock to start */
 792         mdelay(1);
 793         /*
 794          * 5. Program the Reset input from automatic test equipment field in the
 795          *    USBP control and status register:
 796          *    USBN_USBP_CTL_STATUS[ATE_RESET] = 1
 797          */
 798         usbn_usbp_ctl_status.u64 = __cvmx_usb_read_csr64(usb,
 799                         CVMX_USBNX_USBP_CTL_STATUS(usb->index));
 800         usbn_usbp_ctl_status.s.ate_reset = 1;
 801         __cvmx_usb_write_csr64(usb, CVMX_USBNX_USBP_CTL_STATUS(usb->index),
 802                                usbn_usbp_ctl_status.u64);
 803         /* 6. Wait 10 cycles */
 804         cvmx_wait(10);
 805         /*
 806          * 7. Clear ATE_RESET field in the USBN clock-control register:
 807          *    USBN_USBP_CTL_STATUS[ATE_RESET] = 0
 808          */
 809         usbn_usbp_ctl_status.s.ate_reset = 0;
 810         __cvmx_usb_write_csr64(usb, CVMX_USBNX_USBP_CTL_STATUS(usb->index),
 811                                usbn_usbp_ctl_status.u64);
 812         /*
 813          * 8. Program the PHY reset field in the USBN clock-control register:
 814          *    USBN_CLK_CTL[PRST] = 1
 815          */
 816         usbn_clk_ctl.s.prst = 1;
 817         __cvmx_usb_write_csr64(usb, CVMX_USBNX_CLK_CTL(usb->index),
 818                                usbn_clk_ctl.u64);
 819         /*
 820          * 9. Program the USBP control and status register to select host or
 821          *    device mode. USBN_USBP_CTL_STATUS[HST_MODE] = 0 for host, = 1 for
 822          *    device
 823          */
 824         usbn_usbp_ctl_status.s.hst_mode = 0;
 825         __cvmx_usb_write_csr64(usb, CVMX_USBNX_USBP_CTL_STATUS(usb->index),
 826                                usbn_usbp_ctl_status.u64);
 827         /* 10. Wait 1 us */
 828         udelay(1);
 829         /*
 830          * 11. Program the hreset_n field in the USBN clock-control register:
 831          *     USBN_CLK_CTL[HRST] = 1
 832          */
 833         usbn_clk_ctl.s.hrst = 1;
 834         __cvmx_usb_write_csr64(usb, CVMX_USBNX_CLK_CTL(usb->index),
 835                                usbn_clk_ctl.u64);
 836         /* 12. Proceed to USB core initialization */
 837         usbn_clk_ctl.s.enable = 1;
 838         __cvmx_usb_write_csr64(usb, CVMX_USBNX_CLK_CTL(usb->index),
 839                                usbn_clk_ctl.u64);
 840         udelay(1);
 841
 842         /*
 843          * USB Core Initialization
 844          *
 845          * 1. Read USBC_GHWCFG1, USBC_GHWCFG2, USBC_GHWCFG3, USBC_GHWCFG4 to
 846          *    determine USB core configuration parameters.
 847          *
 848          *    Nothing needed
 849          *
 850          * 2. Program the following fields in the global AHB configuration
 851          *    register (USBC_GAHBCFG)
 852          *    DMA mode, USBC_GAHBCFG[DMAEn]: 1 = DMA mode, 0 = slave mode
 853          *    Burst length, USBC_GAHBCFG[HBSTLEN] = 0
 854          *    Nonperiodic TxFIFO empty level (slave mode only),
 855          *    USBC_GAHBCFG[NPTXFEMPLVL]
 856          *    Periodic TxFIFO empty level (slave mode only),
 857          *    USBC_GAHBCFG[PTXFEMPLVL]
 858          *    Global interrupt mask, USBC_GAHBCFG[GLBLINTRMSK] = 1
 859          */
 860         {
 861                 union cvmx_usbcx_gahbcfg usbcx_gahbcfg;
 862                 /* Due to an errata, CN31XX doesn't support DMA */
 863                 if (OCTEON_IS_MODEL(OCTEON_CN31XX))
 864                         usb->init_flags |= CVMX_USB_INITIALIZE_FLAGS_NO_DMA;
 865                 usbcx_gahbcfg.u32 = 0;
 866                 usbcx_gahbcfg.s.dmaen = !(usb->init_flags &
 867                                           CVMX_USB_INITIALIZE_FLAGS_NO_DMA);
 868                 if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)
 869                         /* Only use one channel with non DMA */
 870                         usb->idle_hardware_channels = 0x1;
 871                 else if (OCTEON_IS_MODEL(OCTEON_CN5XXX))
 872                         /* CN5XXX have an errata with channel 3 */
 873                         usb->idle_hardware_channels = 0xf7;
 874                 else
 875                         usb->idle_hardware_channels = 0xff;
 876                 usbcx_gahbcfg.s.hbstlen = 0;
 877                 usbcx_gahbcfg.s.nptxfemplvl = 1;
 878                 usbcx_gahbcfg.s.ptxfemplvl = 1;
 879                 usbcx_gahbcfg.s.glblintrmsk = 1;
 880                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_GAHBCFG(usb->index),
 881                                        usbcx_gahbcfg.u32);
 882         }
 883         /*
 884          * 3. Program the following fields in USBC_GUSBCFG register.
 885          *    HS/FS timeout calibration, USBC_GUSBCFG[TOUTCAL] = 0
 886          *    ULPI DDR select, USBC_GUSBCFG[DDRSEL] = 0
 887          *    USB turnaround time, USBC_GUSBCFG[USBTRDTIM] = 0x5
 888          *    PHY low-power clock select, USBC_GUSBCFG[PHYLPWRCLKSEL] = 0
 889          */
 890         {
 891                 union cvmx_usbcx_gusbcfg usbcx_gusbcfg;
 892
 893                 usbcx_gusbcfg.u32 = __cvmx_usb_read_csr32(usb,
 894                                 CVMX_USBCX_GUSBCFG(usb->index));
 895                 usbcx_gusbcfg.s.toutcal = 0;
 896                 usbcx_gusbcfg.s.ddrsel = 0;
 897                 usbcx_gusbcfg.s.usbtrdtim = 0x5;
 898                 usbcx_gusbcfg.s.phylpwrclksel = 0;
 899                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_GUSBCFG(usb->index),
 900                                        usbcx_gusbcfg.u32);
 901         }
 902         /*
 903          * 4. The software must unmask the following bits in the USBC_GINTMSK
 904          *    register.
 905          *    OTG interrupt mask, USBC_GINTMSK[OTGINTMSK] = 1
 906          *    Mode mismatch interrupt mask, USBC_GINTMSK[MODEMISMSK] = 1
 907          */
 908         {
 909                 union cvmx_usbcx_gintmsk usbcx_gintmsk;
 910                 int channel;
 911
 912                 usbcx_gintmsk.u32 = __cvmx_usb_read_csr32(usb,
 913                                 CVMX_USBCX_GINTMSK(usb->index));
 914                 usbcx_gintmsk.s.otgintmsk = 1;
 915                 usbcx_gintmsk.s.modemismsk = 1;
 916                 usbcx_gintmsk.s.hchintmsk = 1;
 917                 usbcx_gintmsk.s.sofmsk = 0;
 918                 /* We need RX FIFO interrupts if we don't have DMA */
 919                 if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)
 920                         usbcx_gintmsk.s.rxflvlmsk = 1;
 921                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_GINTMSK(usb->index),
 922                                        usbcx_gintmsk.u32);
 923
 924                 /*
 925                  * Disable all channel interrupts. We'll enable them per channel
 926                  * later.
 927                  */
 928                 for (channel = 0; channel < 8; channel++)
 929                         __cvmx_usb_write_csr32(usb,
 930                                 CVMX_USBCX_HCINTMSKX(channel, usb->index), 0);
 931         }
 932
 933         {
 934                 /*
 935                  * Host Port Initialization
 936                  *
 937                  * 1. Program the host-port interrupt-mask field to unmask,
 938                  *    USBC_GINTMSK[PRTINT] = 1
 939                  */
 940                 USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index),
 941                                 union cvmx_usbcx_gintmsk, prtintmsk, 1);
 942                 USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index),
 943                                 union cvmx_usbcx_gintmsk, disconnintmsk, 1);
 944                 /*
 945                  * 2. Program the USBC_HCFG register to select full-speed host
 946                  *    or high-speed host.
 947                  */
 948                 {
 949                         union cvmx_usbcx_hcfg usbcx_hcfg;
 950
 951                         usbcx_hcfg.u32 = __cvmx_usb_read_csr32(usb,
 952                                         CVMX_USBCX_HCFG(usb->index));
 953                         usbcx_hcfg.s.fslssupp = 0;
 954                         usbcx_hcfg.s.fslspclksel = 0;
 955                         __cvmx_usb_write_csr32(usb,
 956                                         CVMX_USBCX_HCFG(usb->index),
 957                                         usbcx_hcfg.u32);
 958                 }
 959                 /*
 960                  * 3. Program the port power bit to drive VBUS on the USB,
 961                  *    USBC_HPRT[PRTPWR] = 1
 962                  */
 963                 USB_SET_FIELD32(CVMX_USBCX_HPRT(usb->index),
 964                                 union cvmx_usbcx_hprt, prtpwr, 1);
 965
 966                 /*
 967                  * Steps 4-15 from the manual are done later in the port enable
 968                  */
 969         }
 970
 971         return 0;
 972 }
 973
 974
 975 /**
 976  * Shutdown a USB port after a call to cvmx_usb_initialize().
 977  * The port should be disabled with all pipes closed when this
 978  * function is called.
 979  *
 980  * @usb: USB device state populated by cvmx_usb_initialize().
 981  *
 982  * Returns: 0 or a negative error code.
 983  */
 984 static int cvmx_usb_shutdown(struct cvmx_usb_state *usb)
 985 {
 986         union cvmx_usbnx_clk_ctl usbn_clk_ctl;
 987
 988         /* Make sure all pipes are closed */
 989         if (!list_empty(&usb->idle_pipes) ||
 990             !list_empty(&usb->active_pipes[CVMX_USB_TRANSFER_ISOCHRONOUS]) ||
 991             !list_empty(&usb->active_pipes[CVMX_USB_TRANSFER_INTERRUPT]) ||
 992             !list_empty(&usb->active_pipes[CVMX_USB_TRANSFER_CONTROL]) ||
 993             !list_empty(&usb->active_pipes[CVMX_USB_TRANSFER_BULK]))
 994                 return -EBUSY;
 995
 996         /* Disable the clocks and put them in power on reset */
 997         usbn_clk_ctl.u64 = __cvmx_usb_read_csr64(usb,
 998                         CVMX_USBNX_CLK_CTL(usb->index));
 999         usbn_clk_ctl.s.enable = 1;
1000         usbn_clk_ctl.s.por = 1;
1001         usbn_clk_ctl.s.hclk_rst = 1;
1002         usbn_clk_ctl.s.prst = 0;
1003         usbn_clk_ctl.s.hrst = 0;
1004         __cvmx_usb_write_csr64(usb, CVMX_USBNX_CLK_CTL(usb->index),
1005                                usbn_clk_ctl.u64);
1006         return 0;
1007 }
1008
1009
1010 /**
1011  * Enable a USB port. After this call succeeds, the USB port is
1012  * online and servicing requests.
1013  *
1014  * @usb: USB device state populated by cvmx_usb_initialize().
1015  *
1016  * Returns: 0 or a negative error code.
1017  */
1018 static int cvmx_usb_enable(struct cvmx_usb_state *usb)
1019 {
1020         union cvmx_usbcx_ghwcfg3 usbcx_ghwcfg3;
1021
1022         usb->usbcx_hprt.u32 = __cvmx_usb_read_csr32(usb,
1023                         CVMX_USBCX_HPRT(usb->index));
1024
1025         /*
1026          * If the port is already enabled the just return. We don't need to do
1027          * anything
1028          */
1029         if (usb->usbcx_hprt.s.prtena)
1030                 return 0;
1031
1032         /* If there is nothing plugged into the port then fail immediately */
1033         if (!usb->usbcx_hprt.s.prtconnsts)
1034                 return -ETIMEDOUT;
1035
1036         /* Program the port reset bit to start the reset process */
1037         USB_SET_FIELD32(CVMX_USBCX_HPRT(usb->index), union cvmx_usbcx_hprt,
1038                         prtrst, 1);
1039
1040         /*
1041          * Wait at least 50ms (high speed), or 10ms (full speed) for the reset
1042          * process to complete.
1043          */
1044         mdelay(50);
1045
1046         /* Program the port reset bit to 0, USBC_HPRT[PRTRST] = 0 */
1047         USB_SET_FIELD32(CVMX_USBCX_HPRT(usb->index), union cvmx_usbcx_hprt,
1048                         prtrst, 0);
1049
1050         /* Wait for the USBC_HPRT[PRTENA]. */
1051         if (CVMX_WAIT_FOR_FIELD32(CVMX_USBCX_HPRT(usb->index),
1052                                 union cvmx_usbcx_hprt, prtena, ==, 1, 100000))
1053                 return -ETIMEDOUT;
1054
1055         /*
1056          * Read the port speed field to get the enumerated speed,
1057          * USBC_HPRT[PRTSPD].
1058          */
1059         usb->usbcx_hprt.u32 = __cvmx_usb_read_csr32(usb,
1060                         CVMX_USBCX_HPRT(usb->index));
1061         usbcx_ghwcfg3.u32 = __cvmx_usb_read_csr32(usb,
1062                         CVMX_USBCX_GHWCFG3(usb->index));
1063
1064         /*
1065          * 13. Program the USBC_GRXFSIZ register to select the size of the
1066          *     receive FIFO (25%).
1067          */
1068         USB_SET_FIELD32(CVMX_USBCX_GRXFSIZ(usb->index),
1069                         union cvmx_usbcx_grxfsiz, rxfdep,
1070                         usbcx_ghwcfg3.s.dfifodepth / 4);
1071         /*
1072          * 14. Program the USBC_GNPTXFSIZ register to select the size and the
1073          *     start address of the non- periodic transmit FIFO for nonperiodic
1074          *     transactions (50%).
1075          */
1076         {
1077                 union cvmx_usbcx_gnptxfsiz siz;
1078
1079                 siz.u32 = __cvmx_usb_read_csr32(usb,
1080                                 CVMX_USBCX_GNPTXFSIZ(usb->index));
1081                 siz.s.nptxfdep = usbcx_ghwcfg3.s.dfifodepth / 2;
1082                 siz.s.nptxfstaddr = usbcx_ghwcfg3.s.dfifodepth / 4;
1083                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_GNPTXFSIZ(usb->index),
1084                                        siz.u32);
1085         }
1086         /*
1087          * 15. Program the USBC_HPTXFSIZ register to select the size and start
1088          *     address of the periodic transmit FIFO for periodic transactions
1089          *     (25%).
1090          */
1091         {
1092                 union cvmx_usbcx_hptxfsiz siz;
1093
1094                 siz.u32 = __cvmx_usb_read_csr32(usb,
1095                                 CVMX_USBCX_HPTXFSIZ(usb->index));
1096                 siz.s.ptxfsize = usbcx_ghwcfg3.s.dfifodepth / 4;
1097                 siz.s.ptxfstaddr = 3 * usbcx_ghwcfg3.s.dfifodepth / 4;
1098                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_HPTXFSIZ(usb->index),
1099                                        siz.u32);
1100         }
1101         /* Flush all FIFOs */
1102         USB_SET_FIELD32(CVMX_USBCX_GRSTCTL(usb->index),
1103                         union cvmx_usbcx_grstctl, txfnum, 0x10);
1104         USB_SET_FIELD32(CVMX_USBCX_GRSTCTL(usb->index),
1105                         union cvmx_usbcx_grstctl, txfflsh, 1);
1106         CVMX_WAIT_FOR_FIELD32(CVMX_USBCX_GRSTCTL(usb->index),
1107                               union cvmx_usbcx_grstctl,
1108                               txfflsh, ==, 0, 100);
1109         USB_SET_FIELD32(CVMX_USBCX_GRSTCTL(usb->index),
1110                         union cvmx_usbcx_grstctl, rxfflsh, 1);
1111         CVMX_WAIT_FOR_FIELD32(CVMX_USBCX_GRSTCTL(usb->index),
1112                               union cvmx_usbcx_grstctl,
1113                               rxfflsh, ==, 0, 100);
1114
1115         return 0;
1116 }
1117
1118
1119 /**
1120  * Disable a USB port. After this call the USB port will not
1121  * generate data transfers and will not generate events.
1122  * Transactions in process will fail and call their
1123  * associated callbacks.
1124  *
1125  * @usb: USB device state populated by cvmx_usb_initialize().
1126  *
1127  * Returns: 0 or a negative error code.
1128  */
1129 static int cvmx_usb_disable(struct cvmx_usb_state *usb)
1130 {
1131         /* Disable the port */
1132         USB_SET_FIELD32(CVMX_USBCX_HPRT(usb->index), union cvmx_usbcx_hprt,
1133                         prtena, 1);
1134         return 0;
1135 }
1136
1137
1138 /**
1139  * Get the current state of the USB port. Use this call to
1140  * determine if the usb port has anything connected, is enabled,
1141  * or has some sort of error condition. The return value of this
1142  * call has "changed" bits to signal of the value of some fields
1143  * have changed between calls.
1144  *
1145  * @usb: USB device state populated by cvmx_usb_initialize().
1146  *
1147  * Returns: Port status information
1148  */
1149 static struct cvmx_usb_port_status cvmx_usb_get_status(
1150                 struct cvmx_usb_state *usb)
1151 {
1152         union cvmx_usbcx_hprt usbc_hprt;
1153         struct cvmx_usb_port_status result;
1154
1155         memset(&result, 0, sizeof(result));
1156
1157         usbc_hprt.u32 = __cvmx_usb_read_csr32(usb,
1158                         CVMX_USBCX_HPRT(usb->index));
1159         result.port_enabled = usbc_hprt.s.prtena;
1160         result.port_over_current = usbc_hprt.s.prtovrcurract;
1161         result.port_powered = usbc_hprt.s.prtpwr;
1162         result.port_speed = usbc_hprt.s.prtspd;
1163         result.connected = usbc_hprt.s.prtconnsts;
1164         result.connect_change =
1165                 (result.connected != usb->port_status.connected);
1166
1167         return result;
1168 }
1169
1170 /**
1171  * Open a virtual pipe between the host and a USB device. A pipe
1172  * must be opened before data can be transferred between a device
1173  * and Octeon.
1174  *
1175  * @usb:             USB device state populated by cvmx_usb_initialize().
1176  * @device_addr:
1177  *                   USB device address to open the pipe to
1178  *                   (0-127).
1179  * @endpoint_num:
1180  *                   USB endpoint number to open the pipe to
1181  *                   (0-15).
1182  * @device_speed:
1183  *                   The speed of the device the pipe is going
1184  *                   to. This must match the device's speed,
1185  *                   which may be different than the port speed.
1186  * @max_packet:      The maximum packet length the device can
1187  *                   transmit/receive (low speed=0-8, full
1188  *                   speed=0-1023, high speed=0-1024). This value
1189  *                   comes from the standard endpoint descriptor
1190  *                   field wMaxPacketSize bits <10:0>.
1191  * @transfer_type:
1192  *                   The type of transfer this pipe is for.
1193  * @transfer_dir:
1194  *                   The direction the pipe is in. This is not
1195  *                   used for control pipes.
1196  * @interval:        For ISOCHRONOUS and INTERRUPT transfers,
1197  *                   this is how often the transfer is scheduled
1198  *                   for. All other transfers should specify
1199  *                   zero. The units are in frames (8000/sec at
1200  *                   high speed, 1000/sec for full speed).
1201  * @multi_count:
1202  *                   For high speed devices, this is the maximum
1203  *                   allowed number of packet per microframe.
1204  *                   Specify zero for non high speed devices. This
1205  *                   value comes from the standard endpoint descriptor
1206  *                   field wMaxPacketSize bits <12:11>.
1207  * @hub_device_addr:
1208  *                   Hub device address this device is connected
1209  *                   to. Devices connected directly to Octeon
1210  *                   use zero. This is only used when the device
1211  *                   is full/low speed behind a high speed hub.
1212  *                   The address will be of the high speed hub,
1213  *                   not and full speed hubs after it.
1214  * @hub_port:        Which port on the hub the device is
1215  *                   connected. Use zero for devices connected
1216  *                   directly to Octeon. Like hub_device_addr,
1217  *                   this is only used for full/low speed
1218  *                   devices behind a high speed hub.
1219  *
1220  * Returns: A non-NULL value is a pipe. NULL means an error.
1221  */
1222 static struct cvmx_usb_pipe *cvmx_usb_open_pipe(struct cvmx_usb_state *usb,
1223                                                 int device_addr,
1224                                                 int endpoint_num,
1225                                                 enum cvmx_usb_speed
1226                                                         device_speed,
1227                                                 int max_packet,
1228                                                 enum cvmx_usb_transfer
1229                                                         transfer_type,
1230                                                 enum cvmx_usb_direction
1231                                                         transfer_dir,
1232                                                 int interval, int multi_count,
1233                                                 int hub_device_addr,
1234                                                 int hub_port)
1235 {
1236         struct cvmx_usb_pipe *pipe;
1237
1238         if (unlikely((device_addr < 0) || (device_addr > MAX_USB_ADDRESS)))
1239                 return NULL;
1240         if (unlikely((endpoint_num < 0) || (endpoint_num > MAX_USB_ENDPOINT)))
1241                 return NULL;
1242         if (unlikely(device_speed > CVMX_USB_SPEED_LOW))
1243                 return NULL;
1244         if (unlikely((max_packet <= 0) || (max_packet > 1024)))
1245                 return NULL;
1246         if (unlikely(transfer_type > CVMX_USB_TRANSFER_INTERRUPT))
1247                 return NULL;
1248         if (unlikely((transfer_dir != CVMX_USB_DIRECTION_OUT) &&
1249                 (transfer_dir != CVMX_USB_DIRECTION_IN)))
1250                 return NULL;
1251         if (unlikely(interval < 0))
1252                 return NULL;
1253         if (unlikely((transfer_type == CVMX_USB_TRANSFER_CONTROL) && interval))
1254                 return NULL;
1255         if (unlikely(multi_count < 0))
1256                 return NULL;
1257         if (unlikely((device_speed != CVMX_USB_SPEED_HIGH) &&
1258                 (multi_count != 0)))
1259                 return NULL;
1260         if (unlikely((hub_device_addr < 0) ||
1261                 (hub_device_addr > MAX_USB_ADDRESS)))
1262                 return NULL;
1263         if (unlikely((hub_port < 0) || (hub_port > MAX_USB_HUB_PORT)))
1264                 return NULL;
1265
1266         pipe = kzalloc(sizeof(*pipe), GFP_ATOMIC);
1267         if (!pipe)
1268                 return NULL;
1269         if ((device_speed == CVMX_USB_SPEED_HIGH) &&
1270                 (transfer_dir == CVMX_USB_DIRECTION_OUT) &&
1271                 (transfer_type == CVMX_USB_TRANSFER_BULK))
1272                 pipe->flags |= __CVMX_USB_PIPE_FLAGS_NEED_PING;
1273         pipe->device_addr = device_addr;
1274         pipe->endpoint_num = endpoint_num;
1275         pipe->device_speed = device_speed;
1276         pipe->max_packet = max_packet;
1277         pipe->transfer_type = transfer_type;
1278         pipe->transfer_dir = transfer_dir;
1279         INIT_LIST_HEAD(&pipe->transactions);
1280
1281         /*
1282          * All pipes use interval to rate limit NAK processing. Force an
1283          * interval if one wasn't supplied
1284          */
1285         if (!interval)
1286                 interval = 1;
1287         if (__cvmx_usb_pipe_needs_split(usb, pipe)) {
1288                 pipe->interval = interval*8;
1289                 /* Force start splits to be schedule on uFrame 0 */
1290                 pipe->next_tx_frame = ((usb->frame_number+7)&~7) +
1291                                         pipe->interval;
1292         } else {
1293                 pipe->interval = interval;
1294                 pipe->next_tx_frame = usb->frame_number + pipe->interval;
1295         }
1296         pipe->multi_count = multi_count;
1297         pipe->hub_device_addr = hub_device_addr;
1298         pipe->hub_port = hub_port;
1299         pipe->pid_toggle = 0;
1300         pipe->split_sc_frame = -1;
1301         list_add_tail(&pipe->node, &usb->idle_pipes);
1302
1303         /*
1304          * We don't need to tell the hardware about this pipe yet since
1305          * it doesn't have any submitted requests
1306          */
1307
1308         return pipe;
1309 }
1310
1311
1312 /**
1313  * Poll the RX FIFOs and remove data as needed. This function is only used
1314  * in non DMA mode. It is very important that this function be called quickly
1315  * enough to prevent FIFO overflow.
1316  *
1317  * @usb:        USB device state populated by cvmx_usb_initialize().
1318  */
1319 static void __cvmx_usb_poll_rx_fifo(struct cvmx_usb_state *usb)
1320 {
1321         union cvmx_usbcx_grxstsph rx_status;
1322         int channel;
1323         int bytes;
1324         uint64_t address;
1325         uint32_t *ptr;
1326
1327         rx_status.u32 = __cvmx_usb_read_csr32(usb,
1328                         CVMX_USBCX_GRXSTSPH(usb->index));
1329         /* Only read data if IN data is there */
1330         if (rx_status.s.pktsts != 2)
1331                 return;
1332         /* Check if no data is available */
1333         if (!rx_status.s.bcnt)
1334                 return;
1335
1336         channel = rx_status.s.chnum;
1337         bytes = rx_status.s.bcnt;
1338         if (!bytes)
1339                 return;
1340
1341         /* Get where the DMA engine would have written this data */
1342         address = __cvmx_usb_read_csr64(usb,
1343                         CVMX_USBNX_DMA0_INB_CHN0(usb->index) + channel*8);
1344
1345         ptr = cvmx_phys_to_ptr(address);
1346         __cvmx_usb_write_csr64(usb,
1347                                CVMX_USBNX_DMA0_INB_CHN0(usb->index) + channel*8,
1348                                address + bytes);
1349
1350         /* Loop writing the FIFO data for this packet into memory */
1351         while (bytes > 0) {
1352                 *ptr++ = __cvmx_usb_read_csr32(usb, USB_FIFO_ADDRESS(channel, usb->index));
1353                 bytes -= 4;
1354         }
1355         CVMX_SYNCW;
1356
1357         return;
1358 }
1359
1360
1361 /**
1362  * Fill the TX hardware fifo with data out of the software
1363  * fifos
1364  *
1365  * @usb:            USB device state populated by cvmx_usb_initialize().
1366  * @fifo:           Software fifo to use
1367  * @available:      Amount of space in the hardware fifo
1368  *
1369  * Returns: Non zero if the hardware fifo was too small and needs
1370  *          to be serviced again.
1371  */
1372 static int __cvmx_usb_fill_tx_hw(struct cvmx_usb_state *usb,
1373                                  struct cvmx_usb_tx_fifo *fifo, int available)
1374 {
1375         /*
1376          * We're done either when there isn't anymore space or the software FIFO
1377          * is empty
1378          */
1379         while (available && (fifo->head != fifo->tail)) {
1380                 int i = fifo->tail;
1381                 const uint32_t *ptr = cvmx_phys_to_ptr(fifo->entry[i].address);
1382                 uint64_t csr_address = USB_FIFO_ADDRESS(fifo->entry[i].channel,
1383                                                         usb->index) ^ 4;
1384                 int words = available;
1385
1386                 /* Limit the amount of data to waht the SW fifo has */
1387                 if (fifo->entry[i].size <= available) {
1388                         words = fifo->entry[i].size;
1389                         fifo->tail++;
1390                         if (fifo->tail > MAX_CHANNELS)
1391                                 fifo->tail = 0;
1392                 }
1393
1394                 /* Update the next locations and counts */
1395                 available -= words;
1396                 fifo->entry[i].address += words * 4;
1397                 fifo->entry[i].size -= words;
1398
1399                 /*
1400                  * Write the HW fifo data. The read every three writes is due
1401                  * to an errata on CN3XXX chips
1402                  */
1403                 while (words > 3) {
1404                         cvmx_write64_uint32(csr_address, *ptr++);
1405                         cvmx_write64_uint32(csr_address, *ptr++);
1406                         cvmx_write64_uint32(csr_address, *ptr++);
1407                         cvmx_read64_uint64(
1408                                         CVMX_USBNX_DMA0_INB_CHN0(usb->index));
1409                         words -= 3;
1410                 }
1411                 cvmx_write64_uint32(csr_address, *ptr++);
1412                 if (--words) {
1413                         cvmx_write64_uint32(csr_address, *ptr++);
1414                         if (--words)
1415                                 cvmx_write64_uint32(csr_address, *ptr++);
1416                 }
1417                 cvmx_read64_uint64(CVMX_USBNX_DMA0_INB_CHN0(usb->index));
1418         }
1419         return fifo->head != fifo->tail;
1420 }
1421
1422
1423 /**
1424  * Check the hardware FIFOs and fill them as needed
1425  *
1426  * @usb:        USB device state populated by cvmx_usb_initialize().
1427  */
1428 static void __cvmx_usb_poll_tx_fifo(struct cvmx_usb_state *usb)
1429 {
1430         if (usb->periodic.head != usb->periodic.tail) {
1431                 union cvmx_usbcx_hptxsts tx_status;
1432
1433                 tx_status.u32 = __cvmx_usb_read_csr32(usb,
1434                                 CVMX_USBCX_HPTXSTS(usb->index));
1435                 if (__cvmx_usb_fill_tx_hw(usb, &usb->periodic,
1436                                           tx_status.s.ptxfspcavail))
1437                         USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index),
1438                                         union cvmx_usbcx_gintmsk,
1439                                         ptxfempmsk, 1);
1440                 else
1441                         USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index),
1442                                         union cvmx_usbcx_gintmsk,
1443                                         ptxfempmsk, 0);
1444         }
1445
1446         if (usb->nonperiodic.head != usb->nonperiodic.tail) {
1447                 union cvmx_usbcx_gnptxsts tx_status;
1448
1449                 tx_status.u32 = __cvmx_usb_read_csr32(usb,
1450                                 CVMX_USBCX_GNPTXSTS(usb->index));
1451                 if (__cvmx_usb_fill_tx_hw(usb, &usb->nonperiodic,
1452                                           tx_status.s.nptxfspcavail))
1453                         USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index),
1454                                         union cvmx_usbcx_gintmsk,
1455                                         nptxfempmsk, 1);
1456                 else
1457                         USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index),
1458                                         union cvmx_usbcx_gintmsk,
1459                                         nptxfempmsk, 0);
1460         }
1461
1462         return;
1463 }
1464
1465
1466 /**
1467  * Fill the TX FIFO with an outgoing packet
1468  *
1469  * @usb:          USB device state populated by cvmx_usb_initialize().
1470  * @channel:      Channel number to get packet from
1471  */
1472 static void __cvmx_usb_fill_tx_fifo(struct cvmx_usb_state *usb, int channel)
1473 {
1474         union cvmx_usbcx_hccharx hcchar;
1475         union cvmx_usbcx_hcspltx usbc_hcsplt;
1476         union cvmx_usbcx_hctsizx usbc_hctsiz;
1477         struct cvmx_usb_tx_fifo *fifo;
1478
1479         /* We only need to fill data on outbound channels */
1480         hcchar.u32 = __cvmx_usb_read_csr32(usb,
1481                         CVMX_USBCX_HCCHARX(channel, usb->index));
1482         if (hcchar.s.epdir != CVMX_USB_DIRECTION_OUT)
1483                 return;
1484
1485         /* OUT Splits only have data on the start and not the complete */
1486         usbc_hcsplt.u32 = __cvmx_usb_read_csr32(usb,
1487                         CVMX_USBCX_HCSPLTX(channel, usb->index));
1488         if (usbc_hcsplt.s.spltena && usbc_hcsplt.s.compsplt)
1489                 return;
1490
1491         /*
1492          * Find out how many bytes we need to fill and convert it into 32bit
1493          * words.
1494          */
1495         usbc_hctsiz.u32 = __cvmx_usb_read_csr32(usb,
1496                         CVMX_USBCX_HCTSIZX(channel, usb->index));
1497         if (!usbc_hctsiz.s.xfersize)
1498                 return;
1499
1500         if ((hcchar.s.eptype == CVMX_USB_TRANSFER_INTERRUPT) ||
1501                 (hcchar.s.eptype == CVMX_USB_TRANSFER_ISOCHRONOUS))
1502                 fifo = &usb->periodic;
1503         else
1504                 fifo = &usb->nonperiodic;
1505
1506         fifo->entry[fifo->head].channel = channel;
1507         fifo->entry[fifo->head].address = __cvmx_usb_read_csr64(usb, CVMX_USBNX_DMA0_OUTB_CHN0(usb->index) + channel*8);
1508         fifo->entry[fifo->head].size = (usbc_hctsiz.s.xfersize+3)>>2;
1509         fifo->head++;
1510         if (fifo->head > MAX_CHANNELS)
1511                 fifo->head = 0;
1512
1513         __cvmx_usb_poll_tx_fifo(usb);
1514
1515         return;
1516 }
1517
1518 /**
1519  * Perform channel specific setup for Control transactions. All
1520  * the generic stuff will already have been done in
1521  * __cvmx_usb_start_channel()
1522  *
1523  * @usb:          USB device state populated by cvmx_usb_initialize().
1524  * @channel:      Channel to setup
1525  * @pipe:         Pipe for control transaction
1526  */
1527 static void __cvmx_usb_start_channel_control(struct cvmx_usb_state *usb,
1528                                              int channel,
1529                                              struct cvmx_usb_pipe *pipe)
1530 {
1531         struct cvmx_usb_transaction *transaction =
1532                 list_first_entry(&pipe->transactions, typeof(*transaction),
1533                                  node);
1534         union cvmx_usb_control_header *header =
1535                 cvmx_phys_to_ptr(transaction->control_header);
1536         int bytes_to_transfer = transaction->buffer_length -
1537                 transaction->actual_bytes;
1538         int packets_to_transfer;
1539         union cvmx_usbcx_hctsizx usbc_hctsiz;
1540
1541         usbc_hctsiz.u32 = __cvmx_usb_read_csr32(usb,
1542                         CVMX_USBCX_HCTSIZX(channel, usb->index));
1543
1544         switch (transaction->stage) {
1545         case CVMX_USB_STAGE_NON_CONTROL:
1546         case CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE:
1547                 cvmx_dprintf("%s: ERROR - Non control stage\n", __func__);
1548                 break;
1549         case CVMX_USB_STAGE_SETUP:
1550                 usbc_hctsiz.s.pid = 3; /* Setup */
1551                 bytes_to_transfer = sizeof(*header);
1552                 /* All Control operations start with a setup going OUT */
1553                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
1554                                 union cvmx_usbcx_hccharx, epdir,
1555                                 CVMX_USB_DIRECTION_OUT);
1556                 /*
1557                  * Setup send the control header instead of the buffer data. The
1558                  * buffer data will be used in the next stage
1559                  */
1560                 __cvmx_usb_write_csr64(usb,
1561                         CVMX_USBNX_DMA0_OUTB_CHN0(usb->index) + channel*8,
1562                         transaction->control_header);
1563                 break;
1564         case CVMX_USB_STAGE_SETUP_SPLIT_COMPLETE:
1565                 usbc_hctsiz.s.pid = 3; /* Setup */
1566                 bytes_to_transfer = 0;
1567                 /* All Control operations start with a setup going OUT */
1568                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
1569                                 union cvmx_usbcx_hccharx, epdir,
1570                                 CVMX_USB_DIRECTION_OUT);
1571
1572                 USB_SET_FIELD32(CVMX_USBCX_HCSPLTX(channel, usb->index),
1573                                 union cvmx_usbcx_hcspltx, compsplt, 1);
1574                 break;
1575         case CVMX_USB_STAGE_DATA:
1576                 usbc_hctsiz.s.pid = __cvmx_usb_get_data_pid(pipe);
1577                 if (__cvmx_usb_pipe_needs_split(usb, pipe)) {
1578                         if (header->s.request_type & 0x80)
1579                                 bytes_to_transfer = 0;
1580                         else if (bytes_to_transfer > pipe->max_packet)
1581                                 bytes_to_transfer = pipe->max_packet;
1582                 }
1583                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
1584                                 union cvmx_usbcx_hccharx, epdir,
1585                                 ((header->s.request_type & 0x80) ?
1586                                         CVMX_USB_DIRECTION_IN :
1587                                         CVMX_USB_DIRECTION_OUT));
1588                 break;
1589         case CVMX_USB_STAGE_DATA_SPLIT_COMPLETE:
1590                 usbc_hctsiz.s.pid = __cvmx_usb_get_data_pid(pipe);
1591                 if (!(header->s.request_type & 0x80))
1592                         bytes_to_transfer = 0;
1593                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
1594                                 union cvmx_usbcx_hccharx, epdir,
1595                                 ((header->s.request_type & 0x80) ?
1596                                         CVMX_USB_DIRECTION_IN :
1597                                         CVMX_USB_DIRECTION_OUT));
1598                 USB_SET_FIELD32(CVMX_USBCX_HCSPLTX(channel, usb->index),
1599                                 union cvmx_usbcx_hcspltx, compsplt, 1);
1600                 break;
1601         case CVMX_USB_STAGE_STATUS:
1602                 usbc_hctsiz.s.pid = __cvmx_usb_get_data_pid(pipe);
1603                 bytes_to_transfer = 0;
1604                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
1605                                 union cvmx_usbcx_hccharx, epdir,
1606                                 ((header->s.request_type & 0x80) ?
1607                                         CVMX_USB_DIRECTION_OUT :
1608                                         CVMX_USB_DIRECTION_IN));
1609                 break;
1610         case CVMX_USB_STAGE_STATUS_SPLIT_COMPLETE:
1611                 usbc_hctsiz.s.pid = __cvmx_usb_get_data_pid(pipe);
1612                 bytes_to_transfer = 0;
1613                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
1614                                 union cvmx_usbcx_hccharx, epdir,
1615                                 ((header->s.request_type & 0x80) ?
1616                                         CVMX_USB_DIRECTION_OUT :
1617                                         CVMX_USB_DIRECTION_IN));
1618                 USB_SET_FIELD32(CVMX_USBCX_HCSPLTX(channel, usb->index),
1619                                 union cvmx_usbcx_hcspltx, compsplt, 1);
1620                 break;
1621         }
1622
1623         /*
1624          * Make sure the transfer never exceeds the byte limit of the hardware.
1625          * Further bytes will be sent as continued transactions
1626          */
1627         if (bytes_to_transfer > MAX_TRANSFER_BYTES) {
1628                 /* Round MAX_TRANSFER_BYTES to a multiple of out packet size */
1629                 bytes_to_transfer = MAX_TRANSFER_BYTES / pipe->max_packet;
1630                 bytes_to_transfer *= pipe->max_packet;
1631         }
1632
1633         /*
1634          * Calculate the number of packets to transfer. If the length is zero
1635          * we still need to transfer one packet
1636          */
1637         packets_to_transfer = (bytes_to_transfer + pipe->max_packet - 1) /
1638                 pipe->max_packet;
1639         if (packets_to_transfer == 0)
1640                 packets_to_transfer = 1;
1641         else if ((packets_to_transfer > 1) &&
1642                         (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)) {
1643                 /*
1644                  * Limit to one packet when not using DMA. Channels must be
1645                  * restarted between every packet for IN transactions, so there
1646                  * is no reason to do multiple packets in a row
1647                  */
1648                 packets_to_transfer = 1;
1649                 bytes_to_transfer = packets_to_transfer * pipe->max_packet;
1650         } else if (packets_to_transfer > MAX_TRANSFER_PACKETS) {
1651                 /*
1652                  * Limit the number of packet and data transferred to what the
1653                  * hardware can handle
1654                  */
1655                 packets_to_transfer = MAX_TRANSFER_PACKETS;
1656                 bytes_to_transfer = packets_to_transfer * pipe->max_packet;
1657         }
1658
1659         usbc_hctsiz.s.xfersize = bytes_to_transfer;
1660         usbc_hctsiz.s.pktcnt = packets_to_transfer;
1661
1662         __cvmx_usb_write_csr32(usb, CVMX_USBCX_HCTSIZX(channel, usb->index),
1663                                usbc_hctsiz.u32);
1664         return;
1665 }
1666
1667
1668 /**
1669  * Start a channel to perform the pipe's head transaction
1670  *
1671  * @usb:          USB device state populated by cvmx_usb_initialize().
1672  * @channel:      Channel to setup
1673  * @pipe:         Pipe to start
1674  */
1675 static void __cvmx_usb_start_channel(struct cvmx_usb_state *usb,
1676                                      int channel,
1677                                      struct cvmx_usb_pipe *pipe)
1678 {
1679         struct cvmx_usb_transaction *transaction =
1680                 list_first_entry(&pipe->transactions, typeof(*transaction),
1681                                  node);
1682
1683         /* Make sure all writes to the DMA region get flushed */
1684         CVMX_SYNCW;
1685
1686         /* Attach the channel to the pipe */
1687         usb->pipe_for_channel[channel] = pipe;
1688         pipe->channel = channel;
1689         pipe->flags |= __CVMX_USB_PIPE_FLAGS_SCHEDULED;
1690
1691         /* Mark this channel as in use */
1692         usb->idle_hardware_channels &= ~(1<<channel);
1693
1694         /* Enable the channel interrupt bits */
1695         {
1696                 union cvmx_usbcx_hcintx usbc_hcint;
1697                 union cvmx_usbcx_hcintmskx usbc_hcintmsk;
1698                 union cvmx_usbcx_haintmsk usbc_haintmsk;
1699
1700                 /* Clear all channel status bits */
1701                 usbc_hcint.u32 = __cvmx_usb_read_csr32(usb,
1702                                 CVMX_USBCX_HCINTX(channel, usb->index));
1703
1704                 __cvmx_usb_write_csr32(usb,
1705                                        CVMX_USBCX_HCINTX(channel, usb->index),
1706                                        usbc_hcint.u32);
1707
1708                 usbc_hcintmsk.u32 = 0;
1709                 usbc_hcintmsk.s.chhltdmsk = 1;
1710                 if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA) {
1711                         /*
1712                          * Channels need these extra interrupts when we aren't
1713                          * in DMA mode.
1714                          */
1715                         usbc_hcintmsk.s.datatglerrmsk = 1;
1716                         usbc_hcintmsk.s.frmovrunmsk = 1;
1717                         usbc_hcintmsk.s.bblerrmsk = 1;
1718                         usbc_hcintmsk.s.xacterrmsk = 1;
1719                         if (__cvmx_usb_pipe_needs_split(usb, pipe)) {
1720                                 /*
1721                                  * Splits don't generate xfercompl, so we need
1722                                  * ACK and NYET.
1723                                  */
1724                                 usbc_hcintmsk.s.nyetmsk = 1;
1725                                 usbc_hcintmsk.s.ackmsk = 1;
1726                         }
1727                         usbc_hcintmsk.s.nakmsk = 1;
1728                         usbc_hcintmsk.s.stallmsk = 1;
1729                         usbc_hcintmsk.s.xfercomplmsk = 1;
1730                 }
1731                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_HCINTMSKX(channel, usb->index), usbc_hcintmsk.u32);
1732
1733                 /* Enable the channel interrupt to propagate */
1734                 usbc_haintmsk.u32 = __cvmx_usb_read_csr32(usb,
1735                                         CVMX_USBCX_HAINTMSK(usb->index));
1736                 usbc_haintmsk.s.haintmsk |= 1<<channel;
1737                 __cvmx_usb_write_csr32(usb,
1738                                         CVMX_USBCX_HAINTMSK(usb->index),
1739                                         usbc_haintmsk.u32);
1740         }
1741
1742         /* Setup the locations the DMA engines use  */
1743         {
1744                 uint64_t dma_address = transaction->buffer +
1745                                         transaction->actual_bytes;
1746
1747                 if (transaction->type == CVMX_USB_TRANSFER_ISOCHRONOUS)
1748                         dma_address = transaction->buffer +
1749                                         transaction->iso_packets[0].offset +
1750                                         transaction->actual_bytes;
1751
1752                 __cvmx_usb_write_csr64(usb,
1753                         CVMX_USBNX_DMA0_OUTB_CHN0(usb->index) + channel*8,
1754                         dma_address);
1755
1756                 __cvmx_usb_write_csr64(usb,
1757                         CVMX_USBNX_DMA0_INB_CHN0(usb->index) + channel*8,
1758                         dma_address);
1759         }
1760
1761         /* Setup both the size of the transfer and the SPLIT characteristics */
1762         {
1763                 union cvmx_usbcx_hcspltx usbc_hcsplt = {.u32 = 0};
1764                 union cvmx_usbcx_hctsizx usbc_hctsiz = {.u32 = 0};
1765                 int packets_to_transfer;
1766                 int bytes_to_transfer = transaction->buffer_length -
1767                         transaction->actual_bytes;
1768
1769                 /*
1770                  * ISOCHRONOUS transactions store each individual transfer size
1771                  * in the packet structure, not the global buffer_length
1772                  */
1773                 if (transaction->type == CVMX_USB_TRANSFER_ISOCHRONOUS)
1774                         bytes_to_transfer =
1775                                 transaction->iso_packets[0].length -
1776                                 transaction->actual_bytes;
1777
1778                 /*
1779                  * We need to do split transactions when we are talking to non
1780                  * high speed devices that are behind a high speed hub
1781                  */
1782                 if (__cvmx_usb_pipe_needs_split(usb, pipe)) {
1783                         /*
1784                          * On the start split phase (stage is even) record the
1785                          * frame number we will need to send the split complete.
1786                          * We only store the lower two bits since the time ahead
1787                          * can only be two frames
1788                          */
1789                         if ((transaction->stage&1) == 0) {
1790                                 if (transaction->type == CVMX_USB_TRANSFER_BULK)
1791                                         pipe->split_sc_frame =
1792                                                 (usb->frame_number + 1) & 0x7f;
1793                                 else
1794                                         pipe->split_sc_frame =
1795                                                 (usb->frame_number + 2) & 0x7f;
1796                         } else
1797                                 pipe->split_sc_frame = -1;
1798
1799                         usbc_hcsplt.s.spltena = 1;
1800                         usbc_hcsplt.s.hubaddr = pipe->hub_device_addr;
1801                         usbc_hcsplt.s.prtaddr = pipe->hub_port;
1802                         usbc_hcsplt.s.compsplt = (transaction->stage ==
1803                                 CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE);
1804
1805                         /*
1806                          * SPLIT transactions can only ever transmit one data
1807                          * packet so limit the transfer size to the max packet
1808                          * size
1809                          */
1810                         if (bytes_to_transfer > pipe->max_packet)
1811                                 bytes_to_transfer = pipe->max_packet;
1812
1813                         /*
1814                          * ISOCHRONOUS OUT splits are unique in that they limit
1815                          * data transfers to 188 byte chunks representing the
1816                          * begin/middle/end of the data or all
1817                          */
1818                         if (!usbc_hcsplt.s.compsplt &&
1819                                 (pipe->transfer_dir ==
1820                                  CVMX_USB_DIRECTION_OUT) &&
1821                                 (pipe->transfer_type ==
1822                                  CVMX_USB_TRANSFER_ISOCHRONOUS)) {
1823                                 /*
1824                                  * Clear the split complete frame number as
1825                                  * there isn't going to be a split complete
1826                                  */
1827                                 pipe->split_sc_frame = -1;
1828                                 /*
1829                                  * See if we've started this transfer and sent
1830                                  * data
1831                                  */
1832                                 if (transaction->actual_bytes == 0) {
1833                                         /*
1834                                          * Nothing sent yet, this is either a
1835                                          * begin or the entire payload
1836                                          */
1837                                         if (bytes_to_transfer <= 188)
1838                                                 /* Entire payload in one go */
1839                                                 usbc_hcsplt.s.xactpos = 3;
1840                                         else
1841                                                 /* First part of payload */
1842                                                 usbc_hcsplt.s.xactpos = 2;
1843                                 } else {
1844                                         /*
1845                                          * Continuing the previous data, we must
1846                                          * either be in the middle or at the end
1847                                          */
1848                                         if (bytes_to_transfer <= 188)
1849                                                 /* End of payload */
1850                                                 usbc_hcsplt.s.xactpos = 1;
1851                                         else
1852                                                 /* Middle of payload */
1853                                                 usbc_hcsplt.s.xactpos = 0;
1854                                 }
1855                                 /*
1856                                  * Again, the transfer size is limited to 188
1857                                  * bytes
1858                                  */
1859                                 if (bytes_to_transfer > 188)
1860                                         bytes_to_transfer = 188;
1861                         }
1862                 }
1863
1864                 /*
1865                  * Make sure the transfer never exceeds the byte limit of the
1866                  * hardware. Further bytes will be sent as continued
1867                  * transactions
1868                  */
1869                 if (bytes_to_transfer > MAX_TRANSFER_BYTES) {
1870                         /*
1871                          * Round MAX_TRANSFER_BYTES to a multiple of out packet
1872                          * size
1873                          */
1874                         bytes_to_transfer = MAX_TRANSFER_BYTES /
1875                                 pipe->max_packet;
1876                         bytes_to_transfer *= pipe->max_packet;
1877                 }
1878
1879                 /*
1880                  * Calculate the number of packets to transfer. If the length is
1881                  * zero we still need to transfer one packet
1882                  */
1883                 packets_to_transfer =
1884                         (bytes_to_transfer + pipe->max_packet - 1) /
1885                         pipe->max_packet;
1886                 if (packets_to_transfer == 0)
1887                         packets_to_transfer = 1;
1888                 else if ((packets_to_transfer > 1) &&
1889                                 (usb->init_flags &
1890                                  CVMX_USB_INITIALIZE_FLAGS_NO_DMA)) {
1891                         /*
1892                          * Limit to one packet when not using DMA. Channels must
1893                          * be restarted between every packet for IN
1894                          * transactions, so there is no reason to do multiple
1895                          * packets in a row
1896                          */
1897                         packets_to_transfer = 1;
1898                         bytes_to_transfer = packets_to_transfer *
1899                                 pipe->max_packet;
1900                 } else if (packets_to_transfer > MAX_TRANSFER_PACKETS) {
1901                         /*
1902                          * Limit the number of packet and data transferred to
1903                          * what the hardware can handle
1904                          */
1905                         packets_to_transfer = MAX_TRANSFER_PACKETS;
1906                         bytes_to_transfer = packets_to_transfer *
1907                                 pipe->max_packet;
1908                 }
1909
1910                 usbc_hctsiz.s.xfersize = bytes_to_transfer;
1911                 usbc_hctsiz.s.pktcnt = packets_to_transfer;
1912
1913                 /* Update the DATA0/DATA1 toggle */
1914                 usbc_hctsiz.s.pid = __cvmx_usb_get_data_pid(pipe);
1915                 /*
1916                  * High speed pipes may need a hardware ping before they start
1917                  */
1918                 if (pipe->flags & __CVMX_USB_PIPE_FLAGS_NEED_PING)
1919                         usbc_hctsiz.s.dopng = 1;
1920
1921                 __cvmx_usb_write_csr32(usb,
1922                                        CVMX_USBCX_HCSPLTX(channel, usb->index),
1923                                        usbc_hcsplt.u32);
1924                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_HCTSIZX(channel,
1925                                         usb->index), usbc_hctsiz.u32);
1926         }
1927
1928         /* Setup the Host Channel Characteristics Register */
1929         {
1930                 union cvmx_usbcx_hccharx usbc_hcchar = {.u32 = 0};
1931
1932                 /*
1933                  * Set the startframe odd/even properly. This is only used for
1934                  * periodic
1935                  */
1936                 usbc_hcchar.s.oddfrm = usb->frame_number&1;
1937
1938                 /*
1939                  * Set the number of back to back packets allowed by this
1940                  * endpoint. Split transactions interpret "ec" as the number of
1941                  * immediate retries of failure. These retries happen too
1942                  * quickly, so we disable these entirely for splits
1943                  */
1944                 if (__cvmx_usb_pipe_needs_split(usb, pipe))
1945                         usbc_hcchar.s.ec = 1;
1946                 else if (pipe->multi_count < 1)
1947                         usbc_hcchar.s.ec = 1;
1948                 else if (pipe->multi_count > 3)
1949                         usbc_hcchar.s.ec = 3;
1950                 else
1951                         usbc_hcchar.s.ec = pipe->multi_count;
1952
1953                 /* Set the rest of the endpoint specific settings */
1954                 usbc_hcchar.s.devaddr = pipe->device_addr;
1955                 usbc_hcchar.s.eptype = transaction->type;
1956                 usbc_hcchar.s.lspddev =
1957                         (pipe->device_speed == CVMX_USB_SPEED_LOW);
1958                 usbc_hcchar.s.epdir = pipe->transfer_dir;
1959                 usbc_hcchar.s.epnum = pipe->endpoint_num;
1960                 usbc_hcchar.s.mps = pipe->max_packet;
1961                 __cvmx_usb_write_csr32(usb,
1962                                        CVMX_USBCX_HCCHARX(channel, usb->index),
1963                                        usbc_hcchar.u32);
1964         }
1965
1966         /* Do transaction type specific fixups as needed */
1967         switch (transaction->type) {
1968         case CVMX_USB_TRANSFER_CONTROL:
1969                 __cvmx_usb_start_channel_control(usb, channel, pipe);
1970                 break;
1971         case CVMX_USB_TRANSFER_BULK:
1972         case CVMX_USB_TRANSFER_INTERRUPT:
1973                 break;
1974         case CVMX_USB_TRANSFER_ISOCHRONOUS:
1975                 if (!__cvmx_usb_pipe_needs_split(usb, pipe)) {
1976                         /*
1977                          * ISO transactions require different PIDs depending on
1978                          * direction and how many packets are needed
1979                          */
1980                         if (pipe->transfer_dir == CVMX_USB_DIRECTION_OUT) {
1981                                 if (pipe->multi_count < 2) /* Need DATA0 */
1982                                         USB_SET_FIELD32(
1983                                                 CVMX_USBCX_HCTSIZX(channel,
1984                                                                    usb->index),
1985                                                 union cvmx_usbcx_hctsizx,
1986                                                 pid, 0);
1987                                 else /* Need MDATA */
1988                                         USB_SET_FIELD32(
1989                                                 CVMX_USBCX_HCTSIZX(channel,
1990                                                                    usb->index),
1991                                                 union cvmx_usbcx_hctsizx,
1992                                                 pid, 3);
1993                         }
1994                 }
1995                 break;
1996         }
1997         {
1998                 union cvmx_usbcx_hctsizx usbc_hctsiz = {.u32 =
1999                         __cvmx_usb_read_csr32(usb,
2000                                 CVMX_USBCX_HCTSIZX(channel, usb->index))};
2001                 transaction->xfersize = usbc_hctsiz.s.xfersize;
2002                 transaction->pktcnt = usbc_hctsiz.s.pktcnt;
2003         }
2004         /* Remeber when we start a split transaction */
2005         if (__cvmx_usb_pipe_needs_split(usb, pipe))
2006                 usb->active_split = transaction;
2007         USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
2008                         union cvmx_usbcx_hccharx, chena, 1);
2009         if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)
2010                 __cvmx_usb_fill_tx_fifo(usb, channel);
2011         return;
2012 }
2013
2014
2015 /**
2016  * Find a pipe that is ready to be scheduled to hardware.
2017  * @usb:         USB device state populated by cvmx_usb_initialize().
2018  * @list:        Pipe list to search
2019  * @current_frame:
2020  *               Frame counter to use as a time reference.
2021  *
2022  * Returns: Pipe or NULL if none are ready
2023  */
2024 static struct cvmx_usb_pipe *__cvmx_usb_find_ready_pipe(
2025                 struct cvmx_usb_state *usb,
2026                 struct list_head *list,
2027                 uint64_t current_frame)
2028 {
2029         struct cvmx_usb_pipe *pipe;
2030
2031         list_for_each_entry(pipe, list, node) {
2032                 struct cvmx_usb_transaction *t =
2033                         list_first_entry(&pipe->transactions, typeof(*t),
2034                                          node);
2035                 if (!(pipe->flags & __CVMX_USB_PIPE_FLAGS_SCHEDULED) && t &&
2036                         (pipe->next_tx_frame <= current_frame) &&
2037                         ((pipe->split_sc_frame == -1) ||
2038                          ((((int)current_frame - (int)pipe->split_sc_frame)
2039                            & 0x7f) < 0x40)) &&
2040                         (!usb->active_split || (usb->active_split == t))) {
2041                         prefetch(t);
2042                         return pipe;
2043                 }
2044         }
2045         return NULL;
2046 }
2047
2048
2049 /**
2050  * Called whenever a pipe might need to be scheduled to the
2051  * hardware.
2052  *
2053  * @usb:         USB device state populated by cvmx_usb_initialize().
2054  * @is_sof:      True if this schedule was called on a SOF interrupt.
2055  */
2056 static void __cvmx_usb_schedule(struct cvmx_usb_state *usb, int is_sof)
2057 {
2058         int channel;
2059         struct cvmx_usb_pipe *pipe;
2060         int need_sof;
2061         enum cvmx_usb_transfer ttype;
2062
2063         if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA) {
2064                 /*
2065                  * Without DMA we need to be careful to not schedule something
2066                  * at the end of a frame and cause an overrun.
2067                  */
2068                 union cvmx_usbcx_hfnum hfnum = {
2069                         .u32 = __cvmx_usb_read_csr32(usb,
2070                                                 CVMX_USBCX_HFNUM(usb->index))
2071                 };
2072
2073                 union cvmx_usbcx_hfir hfir = {
2074                         .u32 = __cvmx_usb_read_csr32(usb,
2075                                                 CVMX_USBCX_HFIR(usb->index))
2076                 };
2077
2078                 if (hfnum.s.frrem < hfir.s.frint/4)
2079                         goto done;
2080         }
2081
2082         while (usb->idle_hardware_channels) {
2083                 /* Find an idle channel */
2084                 channel = __fls(usb->idle_hardware_channels);
2085                 if (unlikely(channel > 7))
2086                         break;
2087
2088                 /* Find a pipe needing service */
2089                 pipe = NULL;
2090                 if (is_sof) {
2091                         /*
2092                          * Only process periodic pipes on SOF interrupts. This
2093                          * way we are sure that the periodic data is sent in the
2094                          * beginning of the frame
2095                          */
2096                         pipe = __cvmx_usb_find_ready_pipe(usb,
2097                                         usb->active_pipes +
2098                                         CVMX_USB_TRANSFER_ISOCHRONOUS,
2099                                         usb->frame_number);
2100                         if (likely(!pipe))
2101                                 pipe = __cvmx_usb_find_ready_pipe(usb,
2102                                                 usb->active_pipes +
2103                                                 CVMX_USB_TRANSFER_INTERRUPT,
2104                                                 usb->frame_number);
2105                 }
2106                 if (likely(!pipe)) {
2107                         pipe = __cvmx_usb_find_ready_pipe(usb,
2108                                         usb->active_pipes +
2109                                         CVMX_USB_TRANSFER_CONTROL,
2110                                         usb->frame_number);
2111                         if (likely(!pipe))
2112                                 pipe = __cvmx_usb_find_ready_pipe(usb,
2113                                                 usb->active_pipes +
2114                                                 CVMX_USB_TRANSFER_BULK,
2115                                                 usb->frame_number);
2116                 }
2117                 if (!pipe)
2118                         break;
2119
2120                 __cvmx_usb_start_channel(usb, channel, pipe);
2121         }
2122
2123 done:
2124         /*
2125          * Only enable SOF interrupts when we have transactions pending in the
2126          * future that might need to be scheduled
2127          */
2128         need_sof = 0;
2129         for (ttype = CVMX_USB_TRANSFER_CONTROL;
2130                         ttype <= CVMX_USB_TRANSFER_INTERRUPT; ttype++) {
2131                 list_for_each_entry(pipe, &usb->active_pipes[ttype], node) {
2132                         if (pipe->next_tx_frame > usb->frame_number) {
2133                                 need_sof = 1;
2134                                 break;
2135                         }
2136                 }
2137         }
2138         USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index),
2139                         union cvmx_usbcx_gintmsk, sofmsk, need_sof);
2140         return;
2141 }
2142
2143 static inline struct octeon_hcd *cvmx_usb_to_octeon(struct cvmx_usb_state *p)
2144 {
2145         return container_of(p, struct octeon_hcd, usb);
2146 }
2147
2148 static inline struct usb_hcd *octeon_to_hcd(struct octeon_hcd *p)
2149 {
2150         return container_of((void *)p, struct usb_hcd, hcd_priv);
2151 }
2152
2153 static void octeon_usb_urb_complete_callback(struct cvmx_usb_state *usb,
2154                                              enum cvmx_usb_complete status,
2155                                              struct cvmx_usb_pipe *pipe,
2156                                              struct cvmx_usb_transaction
2157                                                 *transaction,
2158                                              int bytes_transferred,
2159                                              struct urb *urb)
2160 {
2161         struct octeon_hcd *priv = cvmx_usb_to_octeon(usb);
2162         struct usb_hcd *hcd = octeon_to_hcd(priv);
2163         struct device *dev = hcd->self.controller;
2164
2165         if (likely(status == CVMX_USB_COMPLETE_SUCCESS))
2166                 urb->actual_length = bytes_transferred;
2167         else
2168                 urb->actual_length = 0;
2169
2170         urb->hcpriv = NULL;
2171
2172         /* For Isochronous transactions we need to update the URB packet status
2173            list from data in our private copy */
2174         if (usb_pipetype(urb->pipe) == PIPE_ISOCHRONOUS) {
2175                 int i;
2176                 /*
2177                  * The pointer to the private list is stored in the setup_packet
2178                  * field.
2179                  */
2180                 struct cvmx_usb_iso_packet *iso_packet =
2181                         (struct cvmx_usb_iso_packet *) urb->setup_packet;
2182                 /* Recalculate the transfer size by adding up each packet */
2183                 urb->actual_length = 0;
2184                 for (i = 0; i < urb->number_of_packets; i++) {
2185                         if (iso_packet[i].status ==
2186                                         CVMX_USB_COMPLETE_SUCCESS) {
2187                                 urb->iso_frame_desc[i].status = 0;
2188                                 urb->iso_frame_desc[i].actual_length =
2189                                         iso_packet[i].length;
2190                                 urb->actual_length +=
2191                                         urb->iso_frame_desc[i].actual_length;
2192                         } else {
2193                                 dev_dbg(dev, "ISOCHRONOUS packet=%d of %d status=%d pipe=%p transaction=%p size=%d\n",
2194                                         i, urb->number_of_packets,
2195                                         iso_packet[i].status, pipe,
2196                                         transaction, iso_packet[i].length);
2197                                 urb->iso_frame_desc[i].status = -EREMOTEIO;
2198                         }
2199                 }
2200                 /* Free the private list now that we don't need it anymore */
2201                 kfree(iso_packet);
2202                 urb->setup_packet = NULL;
2203         }
2204
2205         switch (status) {
2206         case CVMX_USB_COMPLETE_SUCCESS:
2207                 urb->status = 0;
2208                 break;
2209         case CVMX_USB_COMPLETE_CANCEL:
2210                 if (urb->status == 0)
2211                         urb->status = -ENOENT;
2212                 break;
2213         case CVMX_USB_COMPLETE_STALL:
2214                 dev_dbg(dev, "status=stall pipe=%p transaction=%p size=%d\n",
2215                         pipe, transaction, bytes_transferred);
2216                 urb->status = -EPIPE;
2217                 break;
2218         case CVMX_USB_COMPLETE_BABBLEERR:
2219                 dev_dbg(dev, "status=babble pipe=%p transaction=%p size=%d\n",
2220                         pipe, transaction, bytes_transferred);
2221                 urb->status = -EPIPE;
2222                 break;
2223         case CVMX_USB_COMPLETE_SHORT:
2224                 dev_dbg(dev, "status=short pipe=%p transaction=%p size=%d\n",
2225                         pipe, transaction, bytes_transferred);
2226                 urb->status = -EREMOTEIO;
2227                 break;
2228         case CVMX_USB_COMPLETE_ERROR:
2229         case CVMX_USB_COMPLETE_XACTERR:
2230         case CVMX_USB_COMPLETE_DATATGLERR:
2231         case CVMX_USB_COMPLETE_FRAMEERR:
2232                 dev_dbg(dev, "status=%d pipe=%p transaction=%p size=%d\n",
2233                         status, pipe, transaction, bytes_transferred);
2234                 urb->status = -EPROTO;
2235                 break;
2236         }
2237         usb_hcd_unlink_urb_from_ep(octeon_to_hcd(priv), urb);
2238         spin_unlock(&priv->lock);
2239         usb_hcd_giveback_urb(octeon_to_hcd(priv), urb, urb->status);
2240         spin_lock(&priv->lock);
2241 }
2242
2243 /**
2244  * Signal the completion of a transaction and free it. The
2245  * transaction will be removed from the pipe transaction list.
2246  *
2247  * @usb:         USB device state populated by cvmx_usb_initialize().
2248  * @pipe:        Pipe the transaction is on
2249  * @transaction:
2250  *               Transaction that completed
2251  * @complete_code:
2252  *               Completion code
2253  */
2254 static void __cvmx_usb_perform_complete(
2255                                 struct cvmx_usb_state *usb,
2256                                 struct cvmx_usb_pipe *pipe,
2257                                 struct cvmx_usb_transaction *transaction,
2258                                 enum cvmx_usb_complete complete_code)
2259 {
2260         /* If this was a split then clear our split in progress marker */
2261         if (usb->active_split == transaction)
2262                 usb->active_split = NULL;
2263
2264         /*
2265          * Isochronous transactions need extra processing as they might not be
2266          * done after a single data transfer
2267          */
2268         if (unlikely(transaction->type == CVMX_USB_TRANSFER_ISOCHRONOUS)) {
2269                 /* Update the number of bytes transferred in this ISO packet */
2270                 transaction->iso_packets[0].length = transaction->actual_bytes;
2271                 transaction->iso_packets[0].status = complete_code;
2272
2273                 /*
2274                  * If there are more ISOs pending and we succeeded, schedule the
2275                  * next one
2276                  */
2277                 if ((transaction->iso_number_packets > 1) &&
2278                         (complete_code == CVMX_USB_COMPLETE_SUCCESS)) {
2279                         /* No bytes transferred for this packet as of yet */
2280                         transaction->actual_bytes = 0;
2281                         /* One less ISO waiting to transfer */
2282                         transaction->iso_number_packets--;
2283                         /* Increment to the next location in our packet array */
2284                         transaction->iso_packets++;
2285                         transaction->stage = CVMX_USB_STAGE_NON_CONTROL;
2286                         goto done;
2287                 }
2288         }
2289
2290         /* Remove the transaction from the pipe list */
2291         list_del(&transaction->node);
2292         if (list_empty(&pipe->transactions))
2293                 list_move_tail(&pipe->node, &usb->idle_pipes);
2294         octeon_usb_urb_complete_callback(usb, complete_code, pipe,
2295                                          transaction,
2296                                          transaction->actual_bytes,
2297                                          transaction->urb);
2298         kfree(transaction);
2299 done:
2300         return;
2301 }
2302
2303
2304 /**
2305  * Submit a usb transaction to a pipe. Called for all types
2306  * of transactions.
2307  *
2308  * @usb:
2309  * @pipe:           Which pipe to submit to.
2310  * @type:           Transaction type
2311  * @buffer:         User buffer for the transaction
2312  * @buffer_length:
2313  *                  User buffer's length in bytes
2314  * @control_header:
2315  *                  For control transactions, the 8 byte standard header
2316  * @iso_start_frame:
2317  *                  For ISO transactions, the start frame
2318  * @iso_number_packets:
2319  *                  For ISO, the number of packet in the transaction.
2320  * @iso_packets:
2321  *                  A description of each ISO packet
2322  * @urb:            URB for the callback
2323  *
2324  * Returns: Transaction or NULL on failure.
2325  */
2326 static struct cvmx_usb_transaction *__cvmx_usb_submit_transaction(
2327                                 struct cvmx_usb_state *usb,
2328                                 struct cvmx_usb_pipe *pipe,
2329                                 enum cvmx_usb_transfer type,
2330                                 uint64_t buffer,
2331                                 int buffer_length,
2332                                 uint64_t control_header,
2333                                 int iso_start_frame,
2334                                 int iso_number_packets,
2335                                 struct cvmx_usb_iso_packet *iso_packets,
2336                                 struct urb *urb)
2337 {
2338         struct cvmx_usb_transaction *transaction;
2339
2340         if (unlikely(pipe->transfer_type != type))
2341                 return NULL;
2342
2343         transaction = kzalloc(sizeof(*transaction), GFP_ATOMIC);
2344         if (unlikely(!transaction))
2345                 return NULL;
2346
2347         transaction->type = type;
2348         transaction->buffer = buffer;
2349         transaction->buffer_length = buffer_length;
2350         transaction->control_header = control_header;
2351         /* FIXME: This is not used, implement it. */
2352         transaction->iso_start_frame = iso_start_frame;
2353         transaction->iso_number_packets = iso_number_packets;
2354         transaction->iso_packets = iso_packets;
2355         transaction->urb = urb;
2356         if (transaction->type == CVMX_USB_TRANSFER_CONTROL)
2357                 transaction->stage = CVMX_USB_STAGE_SETUP;
2358         else
2359                 transaction->stage = CVMX_USB_STAGE_NON_CONTROL;
2360
2361         if (!list_empty(&pipe->transactions)) {
2362                 list_add_tail(&transaction->node, &pipe->transactions);
2363         } else {
2364                 list_add_tail(&transaction->node, &pipe->transactions);
2365                 list_move_tail(&pipe->node,
2366                                &usb->active_pipes[pipe->transfer_type]);
2367
2368                 /*
2369                  * We may need to schedule the pipe if this was the head of the
2370                  * pipe.
2371                  */
2372                 __cvmx_usb_schedule(usb, 0);
2373         }
2374
2375         return transaction;
2376 }
2377
2378
2379 /**
2380  * Call to submit a USB Bulk transfer to a pipe.
2381  *
2382  * @usb:            USB device state populated by cvmx_usb_initialize().
2383  * @pipe:           Handle to the pipe for the transfer.
2384  * @urb:            URB.
2385  *
2386  * Returns: A submitted transaction or NULL on failure.
2387  */
2388 static struct cvmx_usb_transaction *cvmx_usb_submit_bulk(
2389                                                 struct cvmx_usb_state *usb,
2390                                                 struct cvmx_usb_pipe *pipe,
2391                                                 struct urb *urb)
2392 {
2393         return __cvmx_usb_submit_transaction(usb, pipe, CVMX_USB_TRANSFER_BULK,
2394                                              urb->transfer_dma,
2395                                              urb->transfer_buffer_length,
2396                                              0, /* control_header */
2397                                              0, /* iso_start_frame */
2398                                              0, /* iso_number_packets */
2399                                              NULL, /* iso_packets */
2400                                              urb);
2401 }
2402
2403
2404 /**
2405  * Call to submit a USB Interrupt transfer to a pipe.
2406  *
2407  * @usb:            USB device state populated by cvmx_usb_initialize().
2408  * @pipe:           Handle to the pipe for the transfer.
2409  * @urb:            URB returned when the callback is called.
2410  *
2411  * Returns: A submitted transaction or NULL on failure.
2412  */
2413 static struct cvmx_usb_transaction *cvmx_usb_submit_interrupt(
2414                                                 struct cvmx_usb_state *usb,
2415                                                 struct cvmx_usb_pipe *pipe,
2416                                                 struct urb *urb)
2417 {
2418         return __cvmx_usb_submit_transaction(usb, pipe,
2419                                              CVMX_USB_TRANSFER_INTERRUPT,
2420                                              urb->transfer_dma,
2421                                              urb->transfer_buffer_length,
2422                                              0, /* control_header */
2423                                              0, /* iso_start_frame */
2424                                              0, /* iso_number_packets */
2425                                              NULL, /* iso_packets */
2426                                              urb);
2427 }
2428
2429
2430 /**
2431  * Call to submit a USB Control transfer to a pipe.
2432  *
2433  * @usb:            USB device state populated by cvmx_usb_initialize().
2434  * @pipe:           Handle to the pipe for the transfer.
2435  * @urb:            URB.
2436  *
2437  * Returns: A submitted transaction or NULL on failure.
2438  */
2439 static struct cvmx_usb_transaction *cvmx_usb_submit_control(
2440                                                 struct cvmx_usb_state *usb,
2441                                                 struct cvmx_usb_pipe *pipe,
2442                                                 struct urb *urb)
2443 {
2444         int buffer_length = urb->transfer_buffer_length;
2445         uint64_t control_header = urb->setup_dma;
2446         union cvmx_usb_control_header *header =
2447                 cvmx_phys_to_ptr(control_header);
2448
2449         if ((header->s.request_type & 0x80) == 0)
2450                 buffer_length = le16_to_cpu(header->s.length);
2451
2452         return __cvmx_usb_submit_transaction(usb, pipe,
2453                                              CVMX_USB_TRANSFER_CONTROL,
2454                                              urb->transfer_dma, buffer_length,
2455                                              control_header,
2456                                              0, /* iso_start_frame */
2457                                              0, /* iso_number_packets */
2458                                              NULL, /* iso_packets */
2459                                              urb);
2460 }
2461
2462
2463 /**
2464  * Call to submit a USB Isochronous transfer to a pipe.
2465  *
2466  * @usb:            USB device state populated by cvmx_usb_initialize().
2467  * @pipe:           Handle to the pipe for the transfer.
2468  * @urb:            URB returned when the callback is called.
2469  *
2470  * Returns: A submitted transaction or NULL on failure.
2471  */
2472 static struct cvmx_usb_transaction *cvmx_usb_submit_isochronous(
2473                                                 struct cvmx_usb_state *usb,
2474                                                 struct cvmx_usb_pipe *pipe,
2475                                                 struct urb *urb)
2476 {
2477         struct cvmx_usb_iso_packet *packets;
2478
2479         packets = (struct cvmx_usb_iso_packet *) urb->setup_packet;
2480         return __cvmx_usb_submit_transaction(usb, pipe,
2481                                              CVMX_USB_TRANSFER_ISOCHRONOUS,
2482                                              urb->transfer_dma,
2483                                              urb->transfer_buffer_length,
2484                                              0, /* control_header */
2485                                              urb->start_frame,
2486                                              urb->number_of_packets,
2487                                              packets, urb);
2488 }
2489
2490
2491 /**
2492  * Cancel one outstanding request in a pipe. Canceling a request
2493  * can fail if the transaction has already completed before cancel
2494  * is called. Even after a successful cancel call, it may take
2495  * a frame or two for the cvmx_usb_poll() function to call the
2496  * associated callback.
2497  *
2498  * @usb:         USB device state populated by cvmx_usb_initialize().
2499  * @pipe:        Pipe to cancel requests in.
2500  * @transaction: Transaction to cancel, returned by the submit function.
2501  *
2502  * Returns: 0 or a negative error code.
2503  */
2504 static int cvmx_usb_cancel(struct cvmx_usb_state *usb,
2505                            struct cvmx_usb_pipe *pipe,
2506                            struct cvmx_usb_transaction *transaction)
2507 {
2508         /*
2509          * If the transaction is the HEAD of the queue and scheduled. We need to
2510          * treat it special
2511          */
2512         if (list_first_entry(&pipe->transactions, typeof(*transaction), node) ==
2513             transaction && (pipe->flags & __CVMX_USB_PIPE_FLAGS_SCHEDULED)) {
2514                 union cvmx_usbcx_hccharx usbc_hcchar;
2515
2516                 usb->pipe_for_channel[pipe->channel] = NULL;
2517                 pipe->flags &= ~__CVMX_USB_PIPE_FLAGS_SCHEDULED;
2518
2519                 CVMX_SYNCW;
2520
2521                 usbc_hcchar.u32 = __cvmx_usb_read_csr32(usb,
2522                                 CVMX_USBCX_HCCHARX(pipe->channel, usb->index));
2523                 /*
2524                  * If the channel isn't enabled then the transaction already
2525                  * completed.
2526                  */
2527                 if (usbc_hcchar.s.chena) {
2528                         usbc_hcchar.s.chdis = 1;
2529                         __cvmx_usb_write_csr32(usb,
2530                                         CVMX_USBCX_HCCHARX(pipe->channel,
2531                                                 usb->index),
2532                                         usbc_hcchar.u32);
2533                 }
2534         }
2535         __cvmx_usb_perform_complete(usb, pipe, transaction,
2536                                     CVMX_USB_COMPLETE_CANCEL);
2537         return 0;
2538 }
2539
2540
2541 /**
2542  * Cancel all outstanding requests in a pipe. Logically all this
2543  * does is call cvmx_usb_cancel() in a loop.
2544  *
2545  * @usb:         USB device state populated by cvmx_usb_initialize().
2546  * @pipe:        Pipe to cancel requests in.
2547  *
2548  * Returns: 0 or a negative error code.
2549  */
2550 static int cvmx_usb_cancel_all(struct cvmx_usb_state *usb,
2551                                struct cvmx_usb_pipe *pipe)
2552 {
2553         struct cvmx_usb_transaction *transaction, *next;
2554
2555         /* Simply loop through and attempt to cancel each transaction */
2556         list_for_each_entry_safe(transaction, next, &pipe->transactions, node) {
2557                 int result = cvmx_usb_cancel(usb, pipe, transaction);
2558
2559                 if (unlikely(result != 0))
2560                         return result;
2561         }
2562         return 0;
2563 }
2564
2565
2566 /**
2567  * Close a pipe created with cvmx_usb_open_pipe().
2568  *
2569  * @usb:         USB device state populated by cvmx_usb_initialize().
2570  * @pipe:        Pipe to close.
2571  *
2572  * Returns: 0 or a negative error code. EBUSY is returned if the pipe has
2573  *          outstanding transfers.
2574  */
2575 static int cvmx_usb_close_pipe(struct cvmx_usb_state *usb,
2576                                struct cvmx_usb_pipe *pipe)
2577 {
2578         /* Fail if the pipe has pending transactions */
2579         if (!list_empty(&pipe->transactions))
2580                 return -EBUSY;
2581
2582         list_del(&pipe->node);
2583         kfree(pipe);
2584
2585         return 0;
2586 }
2587
2588 /**
2589  * Get the current USB protocol level frame number. The frame
2590  * number is always in the range of 0-0x7ff.
2591  *
2592  * @usb: USB device state populated by cvmx_usb_initialize().
2593  *
2594  * Returns: USB frame number
2595  */
2596 static int cvmx_usb_get_frame_number(struct cvmx_usb_state *usb)
2597 {
2598         int frame_number;
2599         union cvmx_usbcx_hfnum usbc_hfnum;
2600
2601         usbc_hfnum.u32 = __cvmx_usb_read_csr32(usb,
2602                         CVMX_USBCX_HFNUM(usb->index));
2603         frame_number = usbc_hfnum.s.frnum;
2604
2605         return frame_number;
2606 }
2607
2608
2609 /**
2610  * Poll a channel for status
2611  *
2612  * @usb:     USB device
2613  * @channel: Channel to poll
2614  *
2615  * Returns: Zero on success
2616  */
2617 static int __cvmx_usb_poll_channel(struct cvmx_usb_state *usb, int channel)
2618 {
2619         union cvmx_usbcx_hcintx usbc_hcint;
2620         union cvmx_usbcx_hctsizx usbc_hctsiz;
2621         union cvmx_usbcx_hccharx usbc_hcchar;
2622         struct cvmx_usb_pipe *pipe;
2623         struct cvmx_usb_transaction *transaction;
2624         int bytes_this_transfer;
2625         int bytes_in_last_packet;
2626         int packets_processed;
2627         int buffer_space_left;
2628
2629         /* Read the interrupt status bits for the channel */
2630         usbc_hcint.u32 = __cvmx_usb_read_csr32(usb,
2631                         CVMX_USBCX_HCINTX(channel, usb->index));
2632
2633         if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA) {
2634                 usbc_hcchar.u32 = __cvmx_usb_read_csr32(usb,
2635                                 CVMX_USBCX_HCCHARX(channel, usb->index));
2636
2637                 if (usbc_hcchar.s.chena && usbc_hcchar.s.chdis) {
2638                         /*
2639                          * There seems to be a bug in CN31XX which can cause
2640                          * interrupt IN transfers to get stuck until we do a
2641                          * write of HCCHARX without changing things
2642                          */
2643                         __cvmx_usb_write_csr32(usb,
2644                                         CVMX_USBCX_HCCHARX(channel,
2645                                                            usb->index),
2646                                         usbc_hcchar.u32);
2647                         return 0;
2648                 }
2649
2650                 /*
2651                  * In non DMA mode the channels don't halt themselves. We need
2652                  * to manually disable channels that are left running
2653                  */
2654                 if (!usbc_hcint.s.chhltd) {
2655                         if (usbc_hcchar.s.chena) {
2656                                 union cvmx_usbcx_hcintmskx hcintmsk;
2657                                 /* Disable all interrupts except CHHLTD */
2658                                 hcintmsk.u32 = 0;
2659                                 hcintmsk.s.chhltdmsk = 1;
2660                                 __cvmx_usb_write_csr32(usb,
2661                                                 CVMX_USBCX_HCINTMSKX(channel,
2662                                                         usb->index),
2663                                                 hcintmsk.u32);
2664                                 usbc_hcchar.s.chdis = 1;
2665                                 __cvmx_usb_write_csr32(usb,
2666                                                 CVMX_USBCX_HCCHARX(channel,
2667                                                         usb->index),
2668                                                 usbc_hcchar.u32);
2669                                 return 0;
2670                         } else if (usbc_hcint.s.xfercompl) {
2671                                 /*
2672                                  * Successful IN/OUT with transfer complete.
2673                                  * Channel halt isn't needed.
2674                                  */
2675                         } else {
2676                                 cvmx_dprintf("USB%d: Channel %d interrupt without halt\n",
2677                                                 usb->index, channel);
2678                                 return 0;
2679                         }
2680                 }
2681         } else {
2682                 /*
2683                  * There is are no interrupts that we need to process when the
2684                  * channel is still running
2685                  */
2686                 if (!usbc_hcint.s.chhltd)
2687                         return 0;
2688         }
2689
2690         /* Disable the channel interrupts now that it is done */
2691         __cvmx_usb_write_csr32(usb,
2692                                 CVMX_USBCX_HCINTMSKX(channel, usb->index),
2693                                 0);
2694         usb->idle_hardware_channels |= (1<<channel);
2695
2696         /* Make sure this channel is tied to a valid pipe */
2697         pipe = usb->pipe_for_channel[channel];
2698         prefetch(pipe);
2699         if (!pipe)
2700                 return 0;
2701         transaction = list_first_entry(&pipe->transactions,
2702                                        typeof(*transaction),
2703                                        node);
2704         prefetch(transaction);
2705
2706         /*
2707          * Disconnect this pipe from the HW channel. Later the schedule
2708          * function will figure out which pipe needs to go
2709          */
2710         usb->pipe_for_channel[channel] = NULL;
2711         pipe->flags &= ~__CVMX_USB_PIPE_FLAGS_SCHEDULED;
2712
2713         /*
2714          * Read the channel config info so we can figure out how much data
2715          * transfered
2716          */
2717         usbc_hcchar.u32 = __cvmx_usb_read_csr32(usb,
2718                         CVMX_USBCX_HCCHARX(channel, usb->index));
2719         usbc_hctsiz.u32 = __cvmx_usb_read_csr32(usb,
2720                         CVMX_USBCX_HCTSIZX(channel, usb->index));
2721
2722         /*
2723          * Calculating the number of bytes successfully transferred is dependent
2724          * on the transfer direction
2725          */
2726         packets_processed = transaction->pktcnt - usbc_hctsiz.s.pktcnt;
2727         if (usbc_hcchar.s.epdir) {
2728                 /*
2729                  * IN transactions are easy. For every byte received the
2730                  * hardware decrements xfersize. All we need to do is subtract
2731                  * the current value of xfersize from its starting value and we
2732                  * know how many bytes were written to the buffer
2733                  */
2734                 bytes_this_transfer = transaction->xfersize -
2735                         usbc_hctsiz.s.xfersize;
2736         } else {
2737                 /*
2738                  * OUT transaction don't decrement xfersize. Instead pktcnt is
2739                  * decremented on every successful packet send. The hardware
2740                  * does this when it receives an ACK, or NYET. If it doesn't
2741                  * receive one of these responses pktcnt doesn't change
2742                  */
2743                 bytes_this_transfer = packets_processed * usbc_hcchar.s.mps;
2744                 /*
2745                  * The last packet may not be a full transfer if we didn't have
2746                  * enough data
2747                  */
2748                 if (bytes_this_transfer > transaction->xfersize)
2749                         bytes_this_transfer = transaction->xfersize;
2750         }
2751         /* Figure out how many bytes were in the last packet of the transfer */
2752         if (packets_processed)
2753                 bytes_in_last_packet = bytes_this_transfer -
2754                         (packets_processed - 1) * usbc_hcchar.s.mps;
2755         else
2756                 bytes_in_last_packet = bytes_this_transfer;
2757
2758         /*
2759          * As a special case, setup transactions output the setup header, not
2760          * the user's data. For this reason we don't count setup data as bytes
2761          * transferred
2762          */
2763         if ((transaction->stage == CVMX_USB_STAGE_SETUP) ||
2764                 (transaction->stage == CVMX_USB_STAGE_SETUP_SPLIT_COMPLETE))
2765                 bytes_this_transfer = 0;
2766
2767         /*
2768          * Add the bytes transferred to the running total. It is important that
2769          * bytes_this_transfer doesn't count any data that needs to be
2770          * retransmitted
2771          */
2772         transaction->actual_bytes += bytes_this_transfer;
2773         if (transaction->type == CVMX_USB_TRANSFER_ISOCHRONOUS)
2774                 buffer_space_left = transaction->iso_packets[0].length -
2775                         transaction->actual_bytes;
2776         else
2777                 buffer_space_left = transaction->buffer_length -
2778                         transaction->actual_bytes;
2779
2780         /*
2781          * We need to remember the PID toggle state for the next transaction.
2782          * The hardware already updated it for the next transaction
2783          */
2784         pipe->pid_toggle = !(usbc_hctsiz.s.pid == 0);
2785
2786         /*
2787          * For high speed bulk out, assume the next transaction will need to do
2788          * a ping before proceeding. If this isn't true the ACK processing below
2789          * will clear this flag
2790          */
2791         if ((pipe->device_speed == CVMX_USB_SPEED_HIGH) &&
2792                 (pipe->transfer_type == CVMX_USB_TRANSFER_BULK) &&
2793                 (pipe->transfer_dir == CVMX_USB_DIRECTION_OUT))
2794                 pipe->flags |= __CVMX_USB_PIPE_FLAGS_NEED_PING;
2795
2796         if (usbc_hcint.s.stall) {
2797                 /*
2798                  * STALL as a response means this transaction cannot be
2799                  * completed because the device can't process transactions. Tell
2800                  * the user. Any data that was transferred will be counted on
2801                  * the actual bytes transferred
2802                  */
2803                 pipe->pid_toggle = 0;
2804                 __cvmx_usb_perform_complete(usb, pipe, transaction,
2805                                             CVMX_USB_COMPLETE_STALL);
2806         } else if (usbc_hcint.s.xacterr) {
2807                 /*
2808                  * We know at least one packet worked if we get a ACK or NAK.
2809                  * Reset the retry counter
2810                  */
2811                 if (usbc_hcint.s.nak || usbc_hcint.s.ack)
2812                         transaction->retries = 0;
2813                 transaction->retries++;
2814                 if (transaction->retries > MAX_RETRIES) {
2815                         /*
2816                          * XactErr as a response means the device signaled
2817                          * something wrong with the transfer. For example, PID
2818                          * toggle errors cause these
2819                          */
2820                         __cvmx_usb_perform_complete(usb, pipe, transaction,
2821                                                     CVMX_USB_COMPLETE_XACTERR);
2822                 } else {
2823                         /*
2824                          * If this was a split then clear our split in progress
2825                          * marker
2826                          */
2827                         if (usb->active_split == transaction)
2828                                 usb->active_split = NULL;
2829                         /*
2830                          * Rewind to the beginning of the transaction by anding
2831                          * off the split complete bit
2832                          */
2833                         transaction->stage &= ~1;
2834                         pipe->split_sc_frame = -1;
2835                         pipe->next_tx_frame += pipe->interval;
2836                         if (pipe->next_tx_frame < usb->frame_number)
2837                                 pipe->next_tx_frame =
2838                                         usb->frame_number + pipe->interval -
2839                                         (usb->frame_number -
2840                                          pipe->next_tx_frame) % pipe->interval;
2841                 }
2842         } else if (usbc_hcint.s.bblerr) {
2843                 /* Babble Error (BblErr) */
2844                 __cvmx_usb_perform_complete(usb, pipe, transaction,
2845                                             CVMX_USB_COMPLETE_BABBLEERR);
2846         } else if (usbc_hcint.s.datatglerr) {
2847                 /* Data toggle error */
2848                 __cvmx_usb_perform_complete(usb, pipe, transaction,
2849                                             CVMX_USB_COMPLETE_DATATGLERR);
2850         } else if (usbc_hcint.s.nyet) {
2851                 /*
2852                  * NYET as a response is only allowed in three cases: as a
2853                  * response to a ping, as a response to a split transaction, and
2854                  * as a response to a bulk out. The ping case is handled by
2855                  * hardware, so we only have splits and bulk out
2856                  */
2857                 if (!__cvmx_usb_pipe_needs_split(usb, pipe)) {
2858                         transaction->retries = 0;
2859                         /*
2860                          * If there is more data to go then we need to try
2861                          * again. Otherwise this transaction is complete
2862                          */
2863                         if ((buffer_space_left == 0) ||
2864                                 (bytes_in_last_packet < pipe->max_packet))
2865                                 __cvmx_usb_perform_complete(usb, pipe,
2866                                                 transaction,
2867                                                 CVMX_USB_COMPLETE_SUCCESS);
2868                 } else {
2869                         /*
2870                          * Split transactions retry the split complete 4 times
2871                          * then rewind to the start split and do the entire
2872                          * transactions again
2873                          */
2874                         transaction->retries++;
2875                         if ((transaction->retries & 0x3) == 0) {
2876                                 /*
2877                                  * Rewind to the beginning of the transaction by
2878                                  * anding off the split complete bit
2879                                  */
2880                                 transaction->stage &= ~1;
2881                                 pipe->split_sc_frame = -1;
2882                         }
2883                 }
2884         } else if (usbc_hcint.s.ack) {
2885                 transaction->retries = 0;
2886                 /*
2887                  * The ACK bit can only be checked after the other error bits.
2888                  * This is because a multi packet transfer may succeed in a
2889                  * number of packets and then get a different response on the
2890                  * last packet. In this case both ACK and the last response bit
2891                  * will be set. If none of the other response bits is set, then
2892                  * the last packet must have been an ACK
2893                  *
2894                  * Since we got an ACK, we know we don't need to do a ping on
2895                  * this pipe
2896                  */
2897                 pipe->flags &= ~__CVMX_USB_PIPE_FLAGS_NEED_PING;
2898
2899                 switch (transaction->type) {
2900                 case CVMX_USB_TRANSFER_CONTROL:
2901                         switch (transaction->stage) {
2902                         case CVMX_USB_STAGE_NON_CONTROL:
2903                         case CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE:
2904                                 /* This should be impossible */
2905                                 __cvmx_usb_perform_complete(usb, pipe,
2906                                         transaction, CVMX_USB_COMPLETE_ERROR);
2907                                 break;
2908                         case CVMX_USB_STAGE_SETUP:
2909                                 pipe->pid_toggle = 1;
2910                                 if (__cvmx_usb_pipe_needs_split(usb, pipe))
2911                                         transaction->stage =
2912                                                 CVMX_USB_STAGE_SETUP_SPLIT_COMPLETE;
2913                                 else {
2914                                         union cvmx_usb_control_header *header =
2915                                                 cvmx_phys_to_ptr(transaction->control_header);
2916                                         if (header->s.length)
2917                                                 transaction->stage = CVMX_USB_STAGE_DATA;
2918                                         else
2919                                                 transaction->stage = CVMX_USB_STAGE_STATUS;
2920                                 }
2921                                 break;
2922                         case CVMX_USB_STAGE_SETUP_SPLIT_COMPLETE:
2923                                 {
2924                                         union cvmx_usb_control_header *header =
2925                                                 cvmx_phys_to_ptr(transaction->control_header);
2926                                         if (header->s.length)
2927                                                 transaction->stage = CVMX_USB_STAGE_DATA;
2928                                         else
2929                                                 transaction->stage = CVMX_USB_STAGE_STATUS;
2930                                 }
2931                                 break;
2932                         case CVMX_USB_STAGE_DATA:
2933                                 if (__cvmx_usb_pipe_needs_split(usb, pipe)) {
2934                                         transaction->stage =
2935                                                 CVMX_USB_STAGE_DATA_SPLIT_COMPLETE;
2936                                         /*
2937                                          * For setup OUT data that are splits,
2938                                          * the hardware doesn't appear to count
2939                                          * transferred data. Here we manually
2940                                          * update the data transferred
2941                                          */
2942                                         if (!usbc_hcchar.s.epdir) {
2943                                                 if (buffer_space_left < pipe->max_packet)
2944                                                         transaction->actual_bytes +=
2945                                                                 buffer_space_left;
2946                                                 else
2947                                                         transaction->actual_bytes +=
2948                                                                 pipe->max_packet;
2949                                         }
2950                                 } else if ((buffer_space_left == 0) ||
2951                                                 (bytes_in_last_packet <
2952                                                  pipe->max_packet)) {
2953                                         pipe->pid_toggle = 1;
2954                                         transaction->stage =
2955                                                 CVMX_USB_STAGE_STATUS;
2956                                 }
2957                                 break;
2958                         case CVMX_USB_STAGE_DATA_SPLIT_COMPLETE:
2959                                 if ((buffer_space_left == 0) ||
2960                                                 (bytes_in_last_packet <
2961                                                  pipe->max_packet)) {
2962                                         pipe->pid_toggle = 1;
2963                                         transaction->stage =
2964                                                 CVMX_USB_STAGE_STATUS;
2965                                 } else {
2966                                         transaction->stage =
2967                                                 CVMX_USB_STAGE_DATA;
2968                                 }
2969                                 break;
2970                         case CVMX_USB_STAGE_STATUS:
2971                                 if (__cvmx_usb_pipe_needs_split(usb, pipe))
2972                                         transaction->stage =
2973                                                 CVMX_USB_STAGE_STATUS_SPLIT_COMPLETE;
2974                                 else
2975                                         __cvmx_usb_perform_complete(usb, pipe,
2976                                                 transaction,
2977                                                 CVMX_USB_COMPLETE_SUCCESS);
2978                                 break;
2979                         case CVMX_USB_STAGE_STATUS_SPLIT_COMPLETE:
2980                                 __cvmx_usb_perform_complete(usb, pipe,
2981                                                 transaction,
2982                                                 CVMX_USB_COMPLETE_SUCCESS);
2983                                 break;
2984                         }
2985                         break;
2986                 case CVMX_USB_TRANSFER_BULK:
2987                 case CVMX_USB_TRANSFER_INTERRUPT:
2988                         /*
2989                          * The only time a bulk transfer isn't complete when it
2990                          * finishes with an ACK is during a split transaction.
2991                          * For splits we need to continue the transfer if more
2992                          * data is needed
2993                          */
2994                         if (__cvmx_usb_pipe_needs_split(usb, pipe)) {
2995                                 if (transaction->stage ==
2996                                                 CVMX_USB_STAGE_NON_CONTROL)
2997                                         transaction->stage =
2998                                                 CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE;
2999                                 else {
3000                                         if (buffer_space_left &&
3001                                                 (bytes_in_last_packet ==
3002                                                  pipe->max_packet))
3003                                                 transaction->stage =
3004                                                         CVMX_USB_STAGE_NON_CONTROL;
3005                                         else {
3006                                                 if (transaction->type ==
3007                                                         CVMX_USB_TRANSFER_INTERRUPT)
3008                                                         pipe->next_tx_frame +=
3009                                                                 pipe->interval;
3010                                                         __cvmx_usb_perform_complete(
3011                                                                 usb,
3012                                                                 pipe,
3013                                                                 transaction,
3014                                                                 CVMX_USB_COMPLETE_SUCCESS);
3015                                         }
3016                                 }
3017                         } else {
3018                                 if ((pipe->device_speed ==
3019                                         CVMX_USB_SPEED_HIGH) &&
3020                                     (pipe->transfer_type ==
3021                                      CVMX_USB_TRANSFER_BULK) &&
3022                                     (pipe->transfer_dir ==
3023                                      CVMX_USB_DIRECTION_OUT) &&
3024                                     (usbc_hcint.s.nak))
3025                                         pipe->flags |=
3026                                                 __CVMX_USB_PIPE_FLAGS_NEED_PING;
3027                                 if (!buffer_space_left ||
3028                                         (bytes_in_last_packet <
3029                                          pipe->max_packet)) {
3030                                         if (transaction->type ==
3031                                                 CVMX_USB_TRANSFER_INTERRUPT)
3032                                                 pipe->next_tx_frame +=
3033                                                         pipe->interval;
3034                                         __cvmx_usb_perform_complete(usb,
3035                                                 pipe,
3036                                                 transaction,
3037                                                 CVMX_USB_COMPLETE_SUCCESS);
3038                                 }
3039                         }
3040                         break;
3041                 case CVMX_USB_TRANSFER_ISOCHRONOUS:
3042                         if (__cvmx_usb_pipe_needs_split(usb, pipe)) {
3043                                 /*
3044                                  * ISOCHRONOUS OUT splits don't require a
3045                                  * complete split stage. Instead they use a
3046                                  * sequence of begin OUT splits to transfer the
3047                                  * data 188 bytes at a time. Once the transfer
3048                                  * is complete, the pipe sleeps until the next
3049                                  * schedule interval
3050                                  */
3051                                 if (pipe->transfer_dir == CVMX_USB_DIRECTION_OUT) {
3052                                         /*
3053                                          * If no space left or this wasn't a max
3054                                          * size packet then this transfer is
3055                                          * complete. Otherwise start it again to
3056                                          * send the next 188 bytes
3057                                          */
3058                                         if (!buffer_space_left ||
3059                                                 (bytes_this_transfer < 188)) {
3060                                                 pipe->next_tx_frame +=
3061                                                         pipe->interval;
3062                                                 __cvmx_usb_perform_complete(
3063                                                         usb,
3064                                                         pipe,
3065                                                         transaction,
3066                                                         CVMX_USB_COMPLETE_SUCCESS);
3067                                         }
3068                                 } else {
3069                                         if (transaction->stage ==
3070                                                 CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE) {
3071                                                 /*
3072                                                  * We are in the incoming data
3073                                                  * phase. Keep getting data
3074                                                  * until we run out of space or
3075                                                  * get a small packet
3076                                                  */
3077                                                 if ((buffer_space_left == 0) ||
3078                                                         (bytes_in_last_packet <
3079                                                          pipe->max_packet)) {
3080                                                         pipe->next_tx_frame +=
3081                                                                 pipe->interval;
3082                                                         __cvmx_usb_perform_complete(
3083                                                                 usb,
3084                                                                 pipe,
3085                                                                 transaction,
3086                                                                 CVMX_USB_COMPLETE_SUCCESS);
3087                                                 }
3088                                         } else
3089                                                 transaction->stage =
3090                                                         CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE;
3091                                 }
3092                         } else {
3093                                 pipe->next_tx_frame += pipe->interval;
3094                                 __cvmx_usb_perform_complete(usb,
3095                                                 pipe,
3096                                                 transaction,
3097                                                 CVMX_USB_COMPLETE_SUCCESS);
3098                         }
3099                         break;
3100                 }
3101         } else if (usbc_hcint.s.nak) {
3102                 /*
3103                  * If this was a split then clear our split in progress marker.
3104                  */
3105                 if (usb->active_split == transaction)
3106                         usb->active_split = NULL;
3107                 /*
3108                  * NAK as a response means the device couldn't accept the
3109                  * transaction, but it should be retried in the future. Rewind
3110                  * to the beginning of the transaction by anding off the split
3111                  * complete bit. Retry in the next interval
3112                  */
3113                 transaction->retries = 0;
3114                 transaction->stage &= ~1;
3115                 pipe->next_tx_frame += pipe->interval;
3116                 if (pipe->next_tx_frame < usb->frame_number)
3117                         pipe->next_tx_frame = usb->frame_number +
3118                                 pipe->interval -
3119                                 (usb->frame_number - pipe->next_tx_frame) %
3120                                 pipe->interval;
3121         } else {
3122                 struct cvmx_usb_port_status port;
3123
3124                 port = cvmx_usb_get_status(usb);
3125                 if (port.port_enabled) {
3126                         /* We'll retry the exact same transaction again */
3127                         transaction->retries++;
3128                 } else {
3129                         /*
3130                          * We get channel halted interrupts with no result bits
3131                          * sets when the cable is unplugged
3132                          */
3133                         __cvmx_usb_perform_complete(usb, pipe, transaction,
3134                                         CVMX_USB_COMPLETE_ERROR);
3135                 }
3136         }
3137         return 0;
3138 }
3139
3140 static void octeon_usb_port_callback(struct cvmx_usb_state *usb)
3141 {
3142         struct octeon_hcd *priv = cvmx_usb_to_octeon(usb);
3143
3144         spin_unlock(&priv->lock);
3145         usb_hcd_poll_rh_status(octeon_to_hcd(priv));
3146         spin_lock(&priv->lock);
3147 }
3148
3149 /**
3150  * Poll the USB block for status and call all needed callback
3151  * handlers. This function is meant to be called in the interrupt
3152  * handler for the USB controller. It can also be called
3153  * periodically in a loop for non-interrupt based operation.
3154  *
3155  * @usb: USB device state populated by cvmx_usb_initialize().
3156  *
3157  * Returns: 0 or a negative error code.
3158  */
3159 static int cvmx_usb_poll(struct cvmx_usb_state *usb)
3160 {
3161         union cvmx_usbcx_hfnum usbc_hfnum;
3162         union cvmx_usbcx_gintsts usbc_gintsts;
3163
3164         prefetch_range(usb, sizeof(*usb));
3165
3166         /* Update the frame counter */
3167         usbc_hfnum.u32 = __cvmx_usb_read_csr32(usb,
3168                                                 CVMX_USBCX_HFNUM(usb->index));
3169         if ((usb->frame_number&0x3fff) > usbc_hfnum.s.frnum)
3170                 usb->frame_number += 0x4000;
3171         usb->frame_number &= ~0x3fffull;
3172         usb->frame_number |= usbc_hfnum.s.frnum;
3173
3174         /* Read the pending interrupts */
3175         usbc_gintsts.u32 = __cvmx_usb_read_csr32(usb,
3176                                                 CVMX_USBCX_GINTSTS(usb->index));
3177
3178         /* Clear the interrupts now that we know about them */
3179         __cvmx_usb_write_csr32(usb,
3180                                 CVMX_USBCX_GINTSTS(usb->index),
3181                                 usbc_gintsts.u32);
3182
3183         if (usbc_gintsts.s.rxflvl) {
3184                 /*
3185                  * RxFIFO Non-Empty (RxFLvl)
3186                  * Indicates that there is at least one packet pending to be
3187                  * read from the RxFIFO.
3188                  *
3189                  * In DMA mode this is handled by hardware
3190                  */
3191                 if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)
3192                         __cvmx_usb_poll_rx_fifo(usb);
3193         }
3194         if (usbc_gintsts.s.ptxfemp || usbc_gintsts.s.nptxfemp) {
3195                 /* Fill the Tx FIFOs when not in DMA mode */
3196                 if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)
3197                         __cvmx_usb_poll_tx_fifo(usb);
3198         }
3199         if (usbc_gintsts.s.disconnint || usbc_gintsts.s.prtint) {
3200                 union cvmx_usbcx_hprt usbc_hprt;
3201                 /*
3202                  * Disconnect Detected Interrupt (DisconnInt)
3203                  * Asserted when a device disconnect is detected.
3204                  *
3205                  * Host Port Interrupt (PrtInt)
3206                  * The core sets this bit to indicate a change in port status of
3207                  * one of the O2P USB core ports in Host mode. The application
3208                  * must read the Host Port Control and Status (HPRT) register to
3209                  * determine the exact event that caused this interrupt. The
3210                  * application must clear the appropriate status bit in the Host
3211                  * Port Control and Status register to clear this bit.
3212                  *
3213                  * Call the user's port callback
3214                  */
3215                 octeon_usb_port_callback(usb);
3216                 /* Clear the port change bits */
3217                 usbc_hprt.u32 = __cvmx_usb_read_csr32(usb,
3218                                 CVMX_USBCX_HPRT(usb->index));
3219                 usbc_hprt.s.prtena = 0;
3220                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_HPRT(usb->index),
3221                                        usbc_hprt.u32);
3222         }
3223         if (usbc_gintsts.s.hchint) {
3224                 /*
3225                  * Host Channels Interrupt (HChInt)
3226                  * The core sets this bit to indicate that an interrupt is
3227                  * pending on one of the channels of the core (in Host mode).
3228                  * The application must read the Host All Channels Interrupt
3229                  * (HAINT) register to determine the exact number of the channel
3230                  * on which the interrupt occurred, and then read the
3231                  * corresponding Host Channel-n Interrupt (HCINTn) register to
3232                  * determine the exact cause of the interrupt. The application
3233                  * must clear the appropriate status bit in the HCINTn register
3234                  * to clear this bit.
3235                  */
3236                 union cvmx_usbcx_haint usbc_haint;
3237
3238                 usbc_haint.u32 = __cvmx_usb_read_csr32(usb,
3239                                         CVMX_USBCX_HAINT(usb->index));
3240                 while (usbc_haint.u32) {
3241                         int channel;
3242
3243                         channel = __fls(usbc_haint.u32);
3244                         __cvmx_usb_poll_channel(usb, channel);
3245                         usbc_haint.u32 ^= 1<<channel;
3246                 }
3247         }
3248
3249         __cvmx_usb_schedule(usb, usbc_gintsts.s.sof);
3250
3251         return 0;
3252 }
3253
3254 /* convert between an HCD pointer and the corresponding struct octeon_hcd */
3255 static inline struct octeon_hcd *hcd_to_octeon(struct usb_hcd *hcd)
3256 {
3257         return (struct octeon_hcd *)(hcd->hcd_priv);
3258 }
3259
3260 static irqreturn_t octeon_usb_irq(struct usb_hcd *hcd)
3261 {
3262         struct octeon_hcd *priv = hcd_to_octeon(hcd);
3263         unsigned long flags;
3264
3265         spin_lock_irqsave(&priv->lock, flags);
3266         cvmx_usb_poll(&priv->usb);
3267         spin_unlock_irqrestore(&priv->lock, flags);
3268         return IRQ_HANDLED;
3269 }
3270
3271 static int octeon_usb_start(struct usb_hcd *hcd)
3272 {
3273         hcd->state = HC_STATE_RUNNING;
3274         return 0;
3275 }
3276
3277 static void octeon_usb_stop(struct usb_hcd *hcd)
3278 {
3279         hcd->state = HC_STATE_HALT;
3280 }
3281
3282 static int octeon_usb_get_frame_number(struct usb_hcd *hcd)
3283 {
3284         struct octeon_hcd *priv = hcd_to_octeon(hcd);
3285
3286         return cvmx_usb_get_frame_number(&priv->usb);
3287 }
3288
3289 static int octeon_usb_urb_enqueue(struct usb_hcd *hcd,
3290                                   struct urb *urb,
3291                                   gfp_t mem_flags)
3292 {
3293         struct octeon_hcd *priv = hcd_to_octeon(hcd);
3294         struct device *dev = hcd->self.controller;
3295         struct cvmx_usb_transaction *transaction = NULL;
3296         struct cvmx_usb_pipe *pipe;
3297         unsigned long flags;
3298         struct cvmx_usb_iso_packet *iso_packet;
3299         struct usb_host_endpoint *ep = urb->ep;
3300         int rc;
3301
3302         urb->status = 0;
3303         spin_lock_irqsave(&priv->lock, flags);
3304
3305         rc = usb_hcd_link_urb_to_ep(hcd, urb);
3306         if (rc) {
3307                 spin_unlock_irqrestore(&priv->lock, flags);
3308                 return rc;
3309         }
3310
3311         if (!ep->hcpriv) {
3312                 enum cvmx_usb_transfer transfer_type;
3313                 enum cvmx_usb_speed speed;
3314                 int split_device = 0;
3315                 int split_port = 0;
3316
3317                 switch (usb_pipetype(urb->pipe)) {
3318                 case PIPE_ISOCHRONOUS:
3319                         transfer_type = CVMX_USB_TRANSFER_ISOCHRONOUS;
3320                         break;
3321                 case PIPE_INTERRUPT:
3322                         transfer_type = CVMX_USB_TRANSFER_INTERRUPT;
3323                         break;
3324                 case PIPE_CONTROL:
3325                         transfer_type = CVMX_USB_TRANSFER_CONTROL;
3326                         break;
3327                 default:
3328                         transfer_type = CVMX_USB_TRANSFER_BULK;
3329                         break;
3330                 }
3331                 switch (urb->dev->speed) {
3332                 case USB_SPEED_LOW:
3333                         speed = CVMX_USB_SPEED_LOW;
3334                         break;
3335                 case USB_SPEED_FULL:
3336                         speed = CVMX_USB_SPEED_FULL;
3337                         break;
3338                 default:
3339                         speed = CVMX_USB_SPEED_HIGH;
3340                         break;
3341                 }
3342                 /*
3343                  * For slow devices on high speed ports we need to find the hub
3344                  * that does the speed translation so we know where to send the
3345                  * split transactions.
3346                  */
3347                 if (speed != CVMX_USB_SPEED_HIGH) {
3348                         /*
3349                          * Start at this device and work our way up the usb
3350                          * tree.
3351                          */
3352                         struct usb_device *dev = urb->dev;
3353
3354                         while (dev->parent) {
3355                                 /*
3356                                  * If our parent is high speed then he'll
3357                                  * receive the splits.
3358                                  */
3359                                 if (dev->parent->speed == USB_SPEED_HIGH) {
3360                                         split_device = dev->parent->devnum;
3361                                         split_port = dev->portnum;
3362                                         break;
3363                                 }
3364                                 /*
3365                                  * Move up the tree one level. If we make it all
3366                                  * the way up the tree, then the port must not
3367                                  * be in high speed mode and we don't need a
3368                                  * split.
3369                                  */
3370                                 dev = dev->parent;
3371                         }
3372                 }
3373                 pipe = cvmx_usb_open_pipe(&priv->usb, usb_pipedevice(urb->pipe),
3374                                           usb_pipeendpoint(urb->pipe), speed,
3375                                           le16_to_cpu(ep->desc.wMaxPacketSize)
3376                                           & 0x7ff,
3377                                           transfer_type,
3378                                           usb_pipein(urb->pipe) ?
3379                                                 CVMX_USB_DIRECTION_IN :
3380                                                 CVMX_USB_DIRECTION_OUT,
3381                                           urb->interval,
3382                                           (le16_to_cpu(ep->desc.wMaxPacketSize)
3383                                            >> 11) & 0x3,
3384                                           split_device, split_port);
3385                 if (!pipe) {
3386                         usb_hcd_unlink_urb_from_ep(hcd, urb);
3387                         spin_unlock_irqrestore(&priv->lock, flags);
3388                         dev_dbg(dev, "Failed to create pipe\n");
3389                         return -ENOMEM;
3390                 }
3391                 ep->hcpriv = pipe;
3392         } else {
3393                 pipe = ep->hcpriv;
3394         }
3395
3396         switch (usb_pipetype(urb->pipe)) {
3397         case PIPE_ISOCHRONOUS:
3398                 dev_dbg(dev, "Submit isochronous to %d.%d\n",
3399                         usb_pipedevice(urb->pipe),
3400                         usb_pipeendpoint(urb->pipe));
3401                 /*
3402                  * Allocate a structure to use for our private list of
3403                  * isochronous packets.
3404                  */
3405                 iso_packet = kmalloc(urb->number_of_packets *
3406                                      sizeof(struct cvmx_usb_iso_packet),
3407                                      GFP_ATOMIC);
3408                 if (iso_packet) {
3409                         int i;
3410                         /* Fill the list with the data from the URB */
3411                         for (i = 0; i < urb->number_of_packets; i++) {
3412                                 iso_packet[i].offset =
3413                                         urb->iso_frame_desc[i].offset;
3414                                 iso_packet[i].length =
3415                                         urb->iso_frame_desc[i].length;
3416                                 iso_packet[i].status =
3417                                         CVMX_USB_COMPLETE_ERROR;
3418                         }
3419                         /*
3420                          * Store a pointer to the list in the URB setup_packet
3421                          * field. We know this currently isn't being used and
3422                          * this saves us a bunch of logic.
3423                          */
3424                         urb->setup_packet = (char *)iso_packet;
3425                         transaction = cvmx_usb_submit_isochronous(&priv->usb,
3426                                                                   pipe, urb);
3427                         /*
3428                          * If submit failed we need to free our private packet
3429                          * list.
3430                          */
3431                         if (!transaction) {
3432                                 urb->setup_packet = NULL;
3433                                 kfree(iso_packet);
3434                         }
3435                 }
3436                 break;
3437         case PIPE_INTERRUPT:
3438                 dev_dbg(dev, "Submit interrupt to %d.%d\n",
3439                         usb_pipedevice(urb->pipe),
3440                         usb_pipeendpoint(urb->pipe));
3441                 transaction = cvmx_usb_submit_interrupt(&priv->usb, pipe, urb);
3442                 break;
3443         case PIPE_CONTROL:
3444                 dev_dbg(dev, "Submit control to %d.%d\n",
3445                         usb_pipedevice(urb->pipe),
3446                         usb_pipeendpoint(urb->pipe));
3447                 transaction = cvmx_usb_submit_control(&priv->usb, pipe, urb);
3448                 break;
3449         case PIPE_BULK:
3450                 dev_dbg(dev, "Submit bulk to %d.%d\n",
3451                         usb_pipedevice(urb->pipe),
3452                         usb_pipeendpoint(urb->pipe));
3453                 transaction = cvmx_usb_submit_bulk(&priv->usb, pipe, urb);
3454                 break;
3455         }
3456         if (!transaction) {
3457                 usb_hcd_unlink_urb_from_ep(hcd, urb);
3458                 spin_unlock_irqrestore(&priv->lock, flags);
3459                 dev_dbg(dev, "Failed to submit\n");
3460                 return -ENOMEM;
3461         }
3462         urb->hcpriv = transaction;
3463         spin_unlock_irqrestore(&priv->lock, flags);
3464         return 0;
3465 }
3466
3467 static int octeon_usb_urb_dequeue(struct usb_hcd *hcd,
3468                                   struct urb *urb,
3469                                   int status)
3470 {
3471         struct octeon_hcd *priv = hcd_to_octeon(hcd);
3472         unsigned long flags;
3473         int rc;
3474
3475         if (!urb->dev)
3476                 return -EINVAL;
3477
3478         spin_lock_irqsave(&priv->lock, flags);
3479
3480         rc = usb_hcd_check_unlink_urb(hcd, urb, status);
3481         if (rc)
3482                 goto out;
3483
3484         urb->status = status;
3485         cvmx_usb_cancel(&priv->usb, urb->ep->hcpriv, urb->hcpriv);
3486
3487 out:
3488         spin_unlock_irqrestore(&priv->lock, flags);
3489
3490         return rc;
3491 }
3492
3493 static void octeon_usb_endpoint_disable(struct usb_hcd *hcd,
3494                                         struct usb_host_endpoint *ep)
3495 {
3496         struct device *dev = hcd->self.controller;
3497
3498         if (ep->hcpriv) {
3499                 struct octeon_hcd *priv = hcd_to_octeon(hcd);
3500                 struct cvmx_usb_pipe *pipe = ep->hcpriv;
3501                 unsigned long flags;
3502
3503                 spin_lock_irqsave(&priv->lock, flags);
3504                 cvmx_usb_cancel_all(&priv->usb, pipe);
3505                 if (cvmx_usb_close_pipe(&priv->usb, pipe))
3506                         dev_dbg(dev, "Closing pipe %p failed\n", pipe);
3507                 spin_unlock_irqrestore(&priv->lock, flags);
3508                 ep->hcpriv = NULL;
3509         }
3510 }
3511
3512 static int octeon_usb_hub_status_data(struct usb_hcd *hcd, char *buf)
3513 {
3514         struct octeon_hcd *priv = hcd_to_octeon(hcd);
3515         struct cvmx_usb_port_status port_status;
3516         unsigned long flags;
3517
3518         spin_lock_irqsave(&priv->lock, flags);
3519         port_status = cvmx_usb_get_status(&priv->usb);
3520         spin_unlock_irqrestore(&priv->lock, flags);
3521         buf[0] = 0;
3522         buf[0] = port_status.connect_change << 1;
3523
3524         return buf[0] != 0;
3525 }
3526
3527 static int octeon_usb_hub_control(struct usb_hcd *hcd, u16 typeReq, u16 wValue,
3528                                 u16 wIndex, char *buf, u16 wLength)
3529 {
3530         struct octeon_hcd *priv = hcd_to_octeon(hcd);
3531         struct device *dev = hcd->self.controller;
3532         struct cvmx_usb_port_status usb_port_status;
3533         int port_status;
3534         struct usb_hub_descriptor *desc;
3535         unsigned long flags;
3536
3537         switch (typeReq) {
3538         case ClearHubFeature:
3539                 dev_dbg(dev, "ClearHubFeature\n");
3540                 switch (wValue) {
3541                 case C_HUB_LOCAL_POWER:
3542                 case C_HUB_OVER_CURRENT:
3543                         /* Nothing required here */
3544                         break;
3545                 default:
3546                         return -EINVAL;
3547                 }
3548                 break;
3549         case ClearPortFeature:
3550                 dev_dbg(dev, "ClearPortFeature\n");
3551                 if (wIndex != 1) {
3552                         dev_dbg(dev, " INVALID\n");
3553                         return -EINVAL;
3554                 }
3555
3556                 switch (wValue) {
3557                 case USB_PORT_FEAT_ENABLE:
3558                         dev_dbg(dev, " ENABLE\n");
3559                         spin_lock_irqsave(&priv->lock, flags);
3560                         cvmx_usb_disable(&priv->usb);
3561                         spin_unlock_irqrestore(&priv->lock, flags);
3562                         break;
3563                 case USB_PORT_FEAT_SUSPEND:
3564                         dev_dbg(dev, " SUSPEND\n");
3565                         /* Not supported on Octeon */
3566                         break;
3567                 case USB_PORT_FEAT_POWER:
3568                         dev_dbg(dev, " POWER\n");
3569                         /* Not supported on Octeon */
3570                         break;
3571                 case USB_PORT_FEAT_INDICATOR:
3572                         dev_dbg(dev, " INDICATOR\n");
3573                         /* Port inidicator not supported */
3574                         break;
3575                 case USB_PORT_FEAT_C_CONNECTION:
3576                         dev_dbg(dev, " C_CONNECTION\n");
3577                         /* Clears drivers internal connect status change flag */
3578                         spin_lock_irqsave(&priv->lock, flags);
3579                         priv->usb.port_status =
3580                                 cvmx_usb_get_status(&priv->usb);
3581                         spin_unlock_irqrestore(&priv->lock, flags);
3582                         break;
3583                 case USB_PORT_FEAT_C_RESET:
3584                         dev_dbg(dev, " C_RESET\n");
3585                         /*
3586                          * Clears the driver's internal Port Reset Change flag.
3587                          */
3588                         spin_lock_irqsave(&priv->lock, flags);
3589                         priv->usb.port_status =
3590                                 cvmx_usb_get_status(&priv->usb);
3591                         spin_unlock_irqrestore(&priv->lock, flags);
3592                         break;
3593                 case USB_PORT_FEAT_C_ENABLE:
3594                         dev_dbg(dev, " C_ENABLE\n");
3595                         /*
3596                          * Clears the driver's internal Port Enable/Disable
3597                          * Change flag.
3598                          */
3599                         spin_lock_irqsave(&priv->lock, flags);
3600                         priv->usb.port_status =
3601                                 cvmx_usb_get_status(&priv->usb);
3602                         spin_unlock_irqrestore(&priv->lock, flags);
3603                         break;
3604                 case USB_PORT_FEAT_C_SUSPEND:
3605                         dev_dbg(dev, " C_SUSPEND\n");
3606                         /*
3607                          * Clears the driver's internal Port Suspend Change
3608                          * flag, which is set when resume signaling on the host
3609                          * port is complete.
3610                          */
3611                         break;
3612                 case USB_PORT_FEAT_C_OVER_CURRENT:
3613                         dev_dbg(dev, " C_OVER_CURRENT\n");
3614                         /* Clears the driver's overcurrent Change flag */
3615                         spin_lock_irqsave(&priv->lock, flags);
3616                         priv->usb.port_status =
3617                                 cvmx_usb_get_status(&priv->usb);
3618                         spin_unlock_irqrestore(&priv->lock, flags);
3619                         break;
3620                 default:
3621                         dev_dbg(dev, " UNKNOWN\n");
3622                         return -EINVAL;
3623                 }
3624                 break;
3625         case GetHubDescriptor:
3626                 dev_dbg(dev, "GetHubDescriptor\n");
3627                 desc = (struct usb_hub_descriptor *)buf;
3628                 desc->bDescLength = 9;
3629                 desc->bDescriptorType = 0x29;
3630                 desc->bNbrPorts = 1;
3631                 desc->wHubCharacteristics = cpu_to_le16(0x08);
3632                 desc->bPwrOn2PwrGood = 1;
3633                 desc->bHubContrCurrent = 0;
3634                 desc->u.hs.DeviceRemovable[0] = 0;
3635                 desc->u.hs.DeviceRemovable[1] = 0xff;
3636                 break;
3637         case GetHubStatus:
3638                 dev_dbg(dev, "GetHubStatus\n");
3639                 *(__le32 *) buf = 0;
3640                 break;
3641         case GetPortStatus:
3642                 dev_dbg(dev, "GetPortStatus\n");
3643                 if (wIndex != 1) {
3644                         dev_dbg(dev, " INVALID\n");
3645                         return -EINVAL;
3646                 }
3647
3648                 spin_lock_irqsave(&priv->lock, flags);
3649                 usb_port_status = cvmx_usb_get_status(&priv->usb);
3650                 spin_unlock_irqrestore(&priv->lock, flags);
3651                 port_status = 0;
3652
3653                 if (usb_port_status.connect_change) {
3654                         port_status |= (1 << USB_PORT_FEAT_C_CONNECTION);
3655                         dev_dbg(dev, " C_CONNECTION\n");
3656                 }
3657
3658                 if (usb_port_status.port_enabled) {
3659                         port_status |= (1 << USB_PORT_FEAT_C_ENABLE);
3660                         dev_dbg(dev, " C_ENABLE\n");
3661                 }
3662
3663                 if (usb_port_status.connected) {
3664                         port_status |= (1 << USB_PORT_FEAT_CONNECTION);
3665                         dev_dbg(dev, " CONNECTION\n");
3666                 }
3667
3668                 if (usb_port_status.port_enabled) {
3669                         port_status |= (1 << USB_PORT_FEAT_ENABLE);
3670                         dev_dbg(dev, " ENABLE\n");
3671                 }
3672
3673                 if (usb_port_status.port_over_current) {
3674                         port_status |= (1 << USB_PORT_FEAT_OVER_CURRENT);
3675                         dev_dbg(dev, " OVER_CURRENT\n");
3676                 }
3677
3678                 if (usb_port_status.port_powered) {
3679                         port_status |= (1 << USB_PORT_FEAT_POWER);
3680                         dev_dbg(dev, " POWER\n");
3681                 }
3682
3683                 if (usb_port_status.port_speed == CVMX_USB_SPEED_HIGH) {
3684                         port_status |= USB_PORT_STAT_HIGH_SPEED;
3685                         dev_dbg(dev, " HIGHSPEED\n");
3686                 } else if (usb_port_status.port_speed == CVMX_USB_SPEED_LOW) {
3687                         port_status |= (1 << USB_PORT_FEAT_LOWSPEED);
3688                         dev_dbg(dev, " LOWSPEED\n");
3689                 }
3690
3691                 *((__le32 *) buf) = cpu_to_le32(port_status);
3692                 break;
3693         case SetHubFeature:
3694                 dev_dbg(dev, "SetHubFeature\n");
3695                 /* No HUB features supported */
3696                 break;
3697         case SetPortFeature:
3698                 dev_dbg(dev, "SetPortFeature\n");
3699                 if (wIndex != 1) {
3700                         dev_dbg(dev, " INVALID\n");
3701                         return -EINVAL;
3702                 }
3703
3704                 switch (wValue) {
3705                 case USB_PORT_FEAT_SUSPEND:
3706                         dev_dbg(dev, " SUSPEND\n");
3707                         return -EINVAL;
3708                 case USB_PORT_FEAT_POWER:
3709                         dev_dbg(dev, " POWER\n");
3710                         return -EINVAL;
3711                 case USB_PORT_FEAT_RESET:
3712                         dev_dbg(dev, " RESET\n");
3713                         spin_lock_irqsave(&priv->lock, flags);
3714                         cvmx_usb_disable(&priv->usb);
3715                         if (cvmx_usb_enable(&priv->usb))
3716                                 dev_dbg(dev, "Failed to enable the port\n");
3717                         spin_unlock_irqrestore(&priv->lock, flags);
3718                         return 0;
3719                 case USB_PORT_FEAT_INDICATOR:
3720                         dev_dbg(dev, " INDICATOR\n");
3721                         /* Not supported */
3722                         break;
3723                 default:
3724                         dev_dbg(dev, " UNKNOWN\n");
3725                         return -EINVAL;
3726                 }
3727                 break;
3728         default:
3729                 dev_dbg(dev, "Unknown root hub request\n");
3730                 return -EINVAL;
3731         }
3732         return 0;
3733 }
3734
3735 static const struct hc_driver octeon_hc_driver = {
3736         .description            = "Octeon USB",
3737         .product_desc           = "Octeon Host Controller",
3738         .hcd_priv_size          = sizeof(struct octeon_hcd),
3739         .irq                    = octeon_usb_irq,
3740         .flags                  = HCD_MEMORY | HCD_USB2,
3741         .start                  = octeon_usb_start,
3742         .stop                   = octeon_usb_stop,
3743         .urb_enqueue            = octeon_usb_urb_enqueue,
3744         .urb_dequeue            = octeon_usb_urb_dequeue,
3745         .endpoint_disable       = octeon_usb_endpoint_disable,
3746         .get_frame_number       = octeon_usb_get_frame_number,
3747         .hub_status_data        = octeon_usb_hub_status_data,
3748         .hub_control            = octeon_usb_hub_control,
3749         .map_urb_for_dma        = octeon_map_urb_for_dma,
3750         .unmap_urb_for_dma      = octeon_unmap_urb_for_dma,
3751 };
3752
3753 static int octeon_usb_probe(struct platform_device *pdev)
3754 {
3755         int status;
3756         int initialize_flags;
3757         int usb_num;
3758         struct resource *res_mem;
3759         struct device_node *usbn_node;
3760         int irq = platform_get_irq(pdev, 0);
3761         struct device *dev = &pdev->dev;
3762         struct octeon_hcd *priv;
3763         struct usb_hcd *hcd;
3764         unsigned long flags;
3765         u32 clock_rate = 48000000;
3766         bool is_crystal_clock = false;
3767         const char *clock_type;
3768         int i;
3769
3770         if (dev->of_node == NULL) {
3771                 dev_err(dev, "Error: empty of_node\n");
3772                 return -ENXIO;
3773         }
3774         usbn_node = dev->of_node->parent;
3775
3776         i = of_property_read_u32(usbn_node,
3777                                  "refclk-frequency", &clock_rate);
3778         if (i) {
3779                 dev_err(dev, "No USBN \"refclk-frequency\"\n");
3780                 return -ENXIO;
3781         }
3782         switch (clock_rate) {
3783         case 12000000:
3784                 initialize_flags = CVMX_USB_INITIALIZE_FLAGS_CLOCK_12MHZ;
3785                 break;
3786         case 24000000:
3787                 initialize_flags = CVMX_USB_INITIALIZE_FLAGS_CLOCK_24MHZ;
3788                 break;
3789         case 48000000:
3790                 initialize_flags = CVMX_USB_INITIALIZE_FLAGS_CLOCK_48MHZ;
3791                 break;
3792         default:
3793                 dev_err(dev, "Illebal USBN \"refclk-frequency\" %u\n",
3794                                 clock_rate);
3795                 return -ENXIO;
3796
3797         }
3798
3799         i = of_property_read_string(usbn_node,
3800                                     "refclk-type", &clock_type);
3801
3802         if (!i && strcmp("crystal", clock_type) == 0)
3803                 is_crystal_clock = true;
3804
3805         if (is_crystal_clock)
3806                 initialize_flags |= CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_XI;
3807         else
3808                 initialize_flags |= CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_GND;
3809
3810         res_mem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
3811         if (res_mem == NULL) {
3812                 dev_err(dev, "found no memory resource\n");
3813                 return -ENXIO;
3814         }
3815         usb_num = (res_mem->start >> 44) & 1;
3816
3817         if (irq < 0) {
3818                 /* Defective device tree, but we know how to fix it. */
3819                 irq_hw_number_t hwirq = usb_num ? (1 << 6) + 17 : 56;
3820
3821                 irq = irq_create_mapping(NULL, hwirq);
3822         }
3823
3824         /*
3825          * Set the DMA mask to 64bits so we get buffers already translated for
3826          * DMA.
3827          */
3828         dev->coherent_dma_mask = ~0;
3829         dev->dma_mask = &dev->coherent_dma_mask;
3830
3831         /*
3832          * Only cn52XX and cn56XX have DWC_OTG USB hardware and the
3833          * IOB priority registers.  Under heavy network load USB
3834          * hardware can be starved by the IOB causing a crash.  Give
3835          * it a priority boost if it has been waiting more than 400
3836          * cycles to avoid this situation.
3837          *
3838          * Testing indicates that a cnt_val of 8192 is not sufficient,
3839          * but no failures are seen with 4096.  We choose a value of
3840          * 400 to give a safety factor of 10.
3841          */
3842         if (OCTEON_IS_MODEL(OCTEON_CN52XX) || OCTEON_IS_MODEL(OCTEON_CN56XX)) {
3843                 union cvmx_iob_n2c_l2c_pri_cnt pri_cnt;
3844
3845                 pri_cnt.u64 = 0;
3846                 pri_cnt.s.cnt_enb = 1;
3847                 pri_cnt.s.cnt_val = 400;
3848                 cvmx_write_csr(CVMX_IOB_N2C_L2C_PRI_CNT, pri_cnt.u64);
3849         }
3850
3851         hcd = usb_create_hcd(&octeon_hc_driver, dev, dev_name(dev));
3852         if (!hcd) {
3853                 dev_dbg(dev, "Failed to allocate memory for HCD\n");
3854                 return -1;
3855         }
3856         hcd->uses_new_polling = 1;
3857         priv = (struct octeon_hcd *)hcd->hcd_priv;
3858
3859         spin_lock_init(&priv->lock);
3860
3861         status = cvmx_usb_initialize(&priv->usb, usb_num, initialize_flags);
3862         if (status) {
3863                 dev_dbg(dev, "USB initialization failed with %d\n", status);
3864                 kfree(hcd);
3865                 return -1;
3866         }
3867
3868         /* This delay is needed for CN3010, but I don't know why... */
3869         mdelay(10);
3870
3871         spin_lock_irqsave(&priv->lock, flags);
3872         cvmx_usb_poll(&priv->usb);
3873         spin_unlock_irqrestore(&priv->lock, flags);
3874
3875         status = usb_add_hcd(hcd, irq, 0);
3876         if (status) {
3877                 dev_dbg(dev, "USB add HCD failed with %d\n", status);
3878                 kfree(hcd);
3879                 return -1;
3880         }
3881         device_wakeup_enable(hcd->self.controller);
3882
3883         dev_info(dev, "Registered HCD for port %d on irq %d\n", usb_num, irq);
3884
3885         return 0;
3886 }
3887
3888 static int octeon_usb_remove(struct platform_device *pdev)
3889 {
3890         int status;
3891         struct device *dev = &pdev->dev;
3892         struct usb_hcd *hcd = dev_get_drvdata(dev);
3893         struct octeon_hcd *priv = hcd_to_octeon(hcd);
3894         unsigned long flags;
3895
3896         usb_remove_hcd(hcd);
3897         spin_lock_irqsave(&priv->lock, flags);
3898         status = cvmx_usb_shutdown(&priv->usb);
3899         spin_unlock_irqrestore(&priv->lock, flags);
3900         if (status)
3901                 dev_dbg(dev, "USB shutdown failed with %d\n", status);
3902
3903         kfree(hcd);
3904
3905         return 0;
3906 }
3907
3908 static struct of_device_id octeon_usb_match[] = {
3909         {
3910                 .compatible = "cavium,octeon-5750-usbc",
3911         },
3912         {},
3913 };
3914
3915 static struct platform_driver octeon_usb_driver = {
3916         .driver = {
3917                 .name       = "OcteonUSB",
3918                 .owner          = THIS_MODULE,
3919                 .of_match_table = octeon_usb_match,
3920         },
3921         .probe      = octeon_usb_probe,
3922         .remove     = octeon_usb_remove,
3923 };
3924
3925 static int __init octeon_usb_driver_init(void)
3926 {
3927         if (usb_disabled())
3928                 return 0;
3929
3930         return platform_driver_register(&octeon_usb_driver);
3931 }
3932 module_init(octeon_usb_driver_init);
3933
3934 static void __exit octeon_usb_driver_exit(void)
3935 {
3936         if (usb_disabled())
3937                 return;
3938
3939         platform_driver_unregister(&octeon_usb_driver);
3940 }
3941 module_exit(octeon_usb_driver_exit);
3942
3943 MODULE_LICENSE("GPL");
3944 MODULE_AUTHOR("Cavium, Inc. <support@cavium.com>");
3945 MODULE_DESCRIPTION("Cavium Inc. OCTEON USB Host driver.");