drivers/char/xillybus/xillybus_core.c

   1 /*
   2  * linux/drivers/misc/xillybus_core.c
   3  *
   4  * Copyright 2011 Xillybus Ltd, http://xillybus.com
   5  *
   6  * Driver for the Xillybus FPGA/host framework.
   7  *
   8  * This driver interfaces with a special IP core in an FPGA, setting up
   9  * a pipe between a hardware FIFO in the programmable logic and a device
  10  * file in the host. The number of such pipes and their attributes are
  11  * set up on the logic. This driver detects these automatically and
  12  * creates the device files accordingly.
  13  *
  14  * This program is free software; you can redistribute it and/or modify
  15  * it under the smems of the GNU General Public License as published by
  16  * the Free Software Foundation; version 2 of the License.
  17  */
  18
  19 #include <linux/list.h>
  20 #include <linux/device.h>
  21 #include <linux/module.h>
  22 #include <linux/io.h>
  23 #include <linux/dma-mapping.h>
  24 #include <linux/interrupt.h>
  25 #include <linux/sched.h>
  26 #include <linux/fs.h>
  27 #include <linux/cdev.h>
  28 #include <linux/spinlock.h>
  29 #include <linux/mutex.h>
  30 #include <linux/crc32.h>
  31 #include <linux/poll.h>
  32 #include <linux/delay.h>
  33 #include <linux/slab.h>
  34 #include <linux/workqueue.h>
  35 #include "xillybus.h"
  36
  37 MODULE_DESCRIPTION("Xillybus core functions");
  38 MODULE_AUTHOR("Eli Billauer, Xillybus Ltd.");
  39 MODULE_VERSION("1.07");
  40 MODULE_ALIAS("xillybus_core");
  41 MODULE_LICENSE("GPL v2");
  42
  43 /* General timeout is 100 ms, rx timeout is 10 ms */
  44 #define XILLY_RX_TIMEOUT (10*HZ/1000)
  45 #define XILLY_TIMEOUT (100*HZ/1000)
  46
  47 #define fpga_msg_ctrl_reg              0x0008
  48 #define fpga_dma_control_reg           0x0020
  49 #define fpga_dma_bufno_reg             0x0024
  50 #define fpga_dma_bufaddr_lowaddr_reg   0x0028
  51 #define fpga_dma_bufaddr_highaddr_reg  0x002c
  52 #define fpga_buf_ctrl_reg              0x0030
  53 #define fpga_buf_offset_reg            0x0034
  54 #define fpga_endian_reg                0x0040
  55
  56 #define XILLYMSG_OPCODE_RELEASEBUF 1
  57 #define XILLYMSG_OPCODE_QUIESCEACK 2
  58 #define XILLYMSG_OPCODE_FIFOEOF 3
  59 #define XILLYMSG_OPCODE_FATAL_ERROR 4
  60 #define XILLYMSG_OPCODE_NONEMPTY 5
  61
  62 static const char xillyname[] = "xillybus";
  63
  64 static struct class *xillybus_class;
  65
  66 /*
  67  * ep_list_lock is the last lock to be taken; No other lock requests are
  68  * allowed while holding it. It merely protects list_of_endpoints, and not
  69  * the endpoints listed in it.
  70  */
  71
  72 static LIST_HEAD(list_of_endpoints);
  73 static struct mutex ep_list_lock;
  74 static struct workqueue_struct *xillybus_wq;
  75
  76 /*
  77  * Locking scheme: Mutexes protect invocations of character device methods.
  78  * If both locks are taken, wr_mutex is taken first, rd_mutex second.
  79  *
  80  * wr_spinlock protects wr_*_buf_idx, wr_empty, wr_sleepy, wr_ready and the
  81  * buffers' end_offset fields against changes made by IRQ handler (and in
  82  * theory, other file request handlers, but the mutex handles that). Nothing
  83  * else.
  84  * They are held for short direct memory manipulations. Needless to say,
  85  * no mutex locking is allowed when a spinlock is held.
  86  *
  87  * rd_spinlock does the same with rd_*_buf_idx, rd_empty and end_offset.
  88  *
  89  * register_mutex is endpoint-specific, and is held when non-atomic
  90  * register operations are performed. wr_mutex and rd_mutex may be
  91  * held when register_mutex is taken, but none of the spinlocks. Note that
  92  * register_mutex doesn't protect against sporadic buf_ctrl_reg writes
  93  * which are unrelated to buf_offset_reg, since they are harmless.
  94  *
  95  * Blocking on the wait queues is allowed with mutexes held, but not with
  96  * spinlocks.
  97  *
  98  * Only interruptible blocking is allowed on mutexes and wait queues.
  99  *
 100  * All in all, the locking order goes (with skips allowed, of course):
 101  * wr_mutex -> rd_mutex -> register_mutex -> wr_spinlock -> rd_spinlock
 102  */
 103
 104 static void malformed_message(struct xilly_endpoint *endpoint, u32 *buf)
 105 {
 106         int opcode;
 107         int msg_channel, msg_bufno, msg_data, msg_dir;
 108
 109         opcode = (buf[0] >> 24) & 0xff;
 110         msg_dir = buf[0] & 1;
 111         msg_channel = (buf[0] >> 1) & 0x7ff;
 112         msg_bufno = (buf[0] >> 12) & 0x3ff;
 113         msg_data = buf[1] & 0xfffffff;
 114
 115         dev_warn(endpoint->dev,
 116                  "Malformed message (skipping): opcode=%d, channel=%03x, dir=%d, bufno=%03x, data=%07x\n",
 117                  opcode, msg_channel, msg_dir, msg_bufno, msg_data);
 118 }
 119
 120 /*
 121  * xillybus_isr assumes the interrupt is allocated exclusively to it,
 122  * which is the natural case MSI and several other hardware-oriented
 123  * interrupts. Sharing is not allowed.
 124  */
 125
 126 irqreturn_t xillybus_isr(int irq, void *data)
 127 {
 128         struct xilly_endpoint *ep = data;
 129         u32 *buf;
 130         unsigned int buf_size;
 131         int i;
 132         int opcode;
 133         unsigned int msg_channel, msg_bufno, msg_data, msg_dir;
 134         struct xilly_channel *channel;
 135
 136         buf = ep->msgbuf_addr;
 137         buf_size = ep->msg_buf_size/sizeof(u32);
 138
 139         ep->ephw->hw_sync_sgl_for_cpu(ep,
 140                                       ep->msgbuf_dma_addr,
 141                                       ep->msg_buf_size,
 142                                       DMA_FROM_DEVICE);
 143
 144         for (i = 0; i < buf_size; i += 2) {
 145                 if (((buf[i+1] >> 28) & 0xf) != ep->msg_counter) {
 146                         malformed_message(ep, &buf[i]);
 147                         dev_warn(ep->dev,
 148                                  "Sending a NACK on counter %x (instead of %x) on entry %d\n",
 149                                  ((buf[i+1] >> 28) & 0xf),
 150                                  ep->msg_counter,
 151                                  i/2);
 152
 153                         if (++ep->failed_messages > 10) {
 154                                 dev_err(ep->dev,
 155                                         "Lost sync with interrupt messages. Stopping.\n");
 156                         } else {
 157                                 ep->ephw->hw_sync_sgl_for_device(
 158                                         ep,
 159                                         ep->msgbuf_dma_addr,
 160                                         ep->msg_buf_size,
 161                                         DMA_FROM_DEVICE);
 162
 163                                 iowrite32(0x01,  /* Message NACK */
 164                                           ep->registers + fpga_msg_ctrl_reg);
 165                         }
 166                         return IRQ_HANDLED;
 167                 } else if (buf[i] & (1 << 22)) /* Last message */
 168                         break;
 169         }
 170
 171         if (i >= buf_size) {
 172                 dev_err(ep->dev, "Bad interrupt message. Stopping.\n");
 173                 return IRQ_HANDLED;
 174         }
 175
 176         buf_size = i + 2;
 177
 178         for (i = 0; i < buf_size; i += 2) { /* Scan through messages */
 179                 opcode = (buf[i] >> 24) & 0xff;
 180
 181                 msg_dir = buf[i] & 1;
 182                 msg_channel = (buf[i] >> 1) & 0x7ff;
 183                 msg_bufno = (buf[i] >> 12) & 0x3ff;
 184                 msg_data = buf[i+1] & 0xfffffff;
 185
 186                 switch (opcode) {
 187                 case XILLYMSG_OPCODE_RELEASEBUF:
 188                         if ((msg_channel > ep->num_channels) ||
 189                             (msg_channel == 0)) {
 190                                 malformed_message(ep, &buf[i]);
 191                                 break;
 192                         }
 193
 194                         channel = ep->channels[msg_channel];
 195
 196                         if (msg_dir) { /* Write channel */
 197                                 if (msg_bufno >= channel->num_wr_buffers) {
 198                                         malformed_message(ep, &buf[i]);
 199                                         break;
 200                                 }
 201                                 spin_lock(&channel->wr_spinlock);
 202                                 channel->wr_buffers[msg_bufno]->end_offset =
 203                                         msg_data;
 204                                 channel->wr_fpga_buf_idx = msg_bufno;
 205                                 channel->wr_empty = 0;
 206                                 channel->wr_sleepy = 0;
 207                                 spin_unlock(&channel->wr_spinlock);
 208
 209                                 wake_up_interruptible(&channel->wr_wait);
 210
 211                         } else {
 212                                 /* Read channel */
 213
 214                                 if (msg_bufno >= channel->num_rd_buffers) {
 215                                         malformed_message(ep, &buf[i]);
 216                                         break;
 217                                 }
 218
 219                                 spin_lock(&channel->rd_spinlock);
 220                                 channel->rd_fpga_buf_idx = msg_bufno;
 221                                 channel->rd_full = 0;
 222                                 spin_unlock(&channel->rd_spinlock);
 223
 224                                 wake_up_interruptible(&channel->rd_wait);
 225                                 if (!channel->rd_synchronous)
 226                                         queue_delayed_work(
 227                                                 xillybus_wq,
 228                                                 &channel->rd_workitem,
 229                                                 XILLY_RX_TIMEOUT);
 230                         }
 231
 232                         break;
 233                 case XILLYMSG_OPCODE_NONEMPTY:
 234                         if ((msg_channel > ep->num_channels) ||
 235                             (msg_channel == 0) || (!msg_dir) ||
 236                             !ep->channels[msg_channel]->wr_supports_nonempty) {
 237                                 malformed_message(ep, &buf[i]);
 238                                 break;
 239                         }
 240
 241                         channel = ep->channels[msg_channel];
 242
 243                         if (msg_bufno >= channel->num_wr_buffers) {
 244                                 malformed_message(ep, &buf[i]);
 245                                 break;
 246                         }
 247                         spin_lock(&channel->wr_spinlock);
 248                         if (msg_bufno == channel->wr_host_buf_idx)
 249                                 channel->wr_ready = 1;
 250                         spin_unlock(&channel->wr_spinlock);
 251
 252                         wake_up_interruptible(&channel->wr_ready_wait);
 253
 254                         break;
 255                 case XILLYMSG_OPCODE_QUIESCEACK:
 256                         ep->idtlen = msg_data;
 257                         wake_up_interruptible(&ep->ep_wait);
 258
 259                         break;
 260                 case XILLYMSG_OPCODE_FIFOEOF:
 261                         if ((msg_channel > ep->num_channels) ||
 262                             (msg_channel == 0) || (!msg_dir) ||
 263                             !ep->channels[msg_channel]->num_wr_buffers) {
 264                                 malformed_message(ep, &buf[i]);
 265                                 break;
 266                         }
 267                         channel = ep->channels[msg_channel];
 268                         spin_lock(&channel->wr_spinlock);
 269                         channel->wr_eof = msg_bufno;
 270                         channel->wr_sleepy = 0;
 271
 272                         channel->wr_hangup = channel->wr_empty &&
 273                                 (channel->wr_host_buf_idx == msg_bufno);
 274
 275                         spin_unlock(&channel->wr_spinlock);
 276
 277                         wake_up_interruptible(&channel->wr_wait);
 278
 279                         break;
 280                 case XILLYMSG_OPCODE_FATAL_ERROR:
 281                         ep->fatal_error = 1;
 282                         wake_up_interruptible(&ep->ep_wait); /* For select() */
 283                         dev_err(ep->dev,
 284                                 "FPGA reported a fatal error. This means that the low-level communication with the device has failed. This hardware problem is most likely unrelated to Xillybus (neither kernel module nor FPGA core), but reports are still welcome. All I/O is aborted.\n");
 285                         break;
 286                 default:
 287                         malformed_message(ep, &buf[i]);
 288                         break;
 289                 }
 290         }
 291
 292         ep->ephw->hw_sync_sgl_for_device(ep,
 293                                          ep->msgbuf_dma_addr,
 294                                          ep->msg_buf_size,
 295                                          DMA_FROM_DEVICE);
 296
 297         ep->msg_counter = (ep->msg_counter + 1) & 0xf;
 298         ep->failed_messages = 0;
 299         iowrite32(0x03, ep->registers + fpga_msg_ctrl_reg); /* Message ACK */
 300
 301         return IRQ_HANDLED;
 302 }
 303 EXPORT_SYMBOL(xillybus_isr);
 304
 305 /*
 306  * A few trivial memory management functions.
 307  * NOTE: These functions are used only on probe and remove, and therefore
 308  * no locks are applied!
 309  */
 310
 311 static void xillybus_autoflush(struct work_struct *work);
 312
 313 struct xilly_alloc_state {
 314         void *salami;
 315         int left_of_salami;
 316         int nbuffer;
 317         enum dma_data_direction direction;
 318         u32 regdirection;
 319 };
 320
 321 static int xilly_get_dma_buffers(struct xilly_endpoint *ep,
 322                                  struct xilly_alloc_state *s,
 323                                  struct xilly_buffer **buffers,
 324                                  int bufnum, int bytebufsize)
 325 {
 326         int i, rc;
 327         dma_addr_t dma_addr;
 328         struct device *dev = ep->dev;
 329         struct xilly_buffer *this_buffer = NULL; /* Init to silence warning */
 330
 331         if (buffers) { /* Not the message buffer */
 332                 this_buffer = devm_kcalloc(dev, bufnum,
 333                                            sizeof(struct xilly_buffer),
 334                                            GFP_KERNEL);
 335                 if (!this_buffer)
 336                         return -ENOMEM;
 337         }
 338
 339         for (i = 0; i < bufnum; i++) {
 340                 /*
 341                  * Buffers are expected in descending size order, so there
 342                  * is either enough space for this buffer or none at all.
 343                  */
 344
 345                 if ((s->left_of_salami < bytebufsize) &&
 346                     (s->left_of_salami > 0)) {
 347                         dev_err(ep->dev,
 348                                 "Corrupt buffer allocation in IDT. Aborting.\n");
 349                         return -ENODEV;
 350                 }
 351
 352                 if (s->left_of_salami == 0) {
 353                         int allocorder, allocsize;
 354
 355                         allocsize = PAGE_SIZE;
 356                         allocorder = 0;
 357                         while (bytebufsize > allocsize) {
 358                                 allocsize *= 2;
 359                                 allocorder++;
 360                         }
 361
 362                         s->salami = (void *) devm_get_free_pages(
 363                                 dev,
 364                                 GFP_KERNEL | __GFP_DMA32 | __GFP_ZERO,
 365                                 allocorder);
 366                         if (!s->salami)
 367                                 return -ENOMEM;
 368
 369                         s->left_of_salami = allocsize;
 370                 }
 371
 372                 rc = ep->ephw->map_single(ep, s->salami,
 373                                           bytebufsize, s->direction,
 374                                           &dma_addr);
 375                 if (rc)
 376                         return rc;
 377
 378                 iowrite32((u32) (dma_addr & 0xffffffff),
 379                           ep->registers + fpga_dma_bufaddr_lowaddr_reg);
 380                 iowrite32(((u32) ((((u64) dma_addr) >> 32) & 0xffffffff)),
 381                           ep->registers + fpga_dma_bufaddr_highaddr_reg);
 382
 383                 if (buffers) { /* Not the message buffer */
 384                         this_buffer->addr = s->salami;
 385                         this_buffer->dma_addr = dma_addr;
 386                         buffers[i] = this_buffer++;
 387
 388                         iowrite32(s->regdirection | s->nbuffer++,
 389                                   ep->registers + fpga_dma_bufno_reg);
 390                 } else {
 391                         ep->msgbuf_addr = s->salami;
 392                         ep->msgbuf_dma_addr = dma_addr;
 393                         ep->msg_buf_size = bytebufsize;
 394
 395                         iowrite32(s->regdirection,
 396                                   ep->registers + fpga_dma_bufno_reg);
 397                 }
 398
 399                 s->left_of_salami -= bytebufsize;
 400                 s->salami += bytebufsize;
 401         }
 402         return 0;
 403 }
 404
 405 static int xilly_setupchannels(struct xilly_endpoint *ep,
 406                                unsigned char *chandesc,
 407                                int entries)
 408 {
 409         struct device *dev = ep->dev;
 410         int i, entry, rc;
 411         struct xilly_channel *channel;
 412         int channelnum, bufnum, bufsize, format, is_writebuf;
 413         int bytebufsize;
 414         int synchronous, allowpartial, exclusive_open, seekable;
 415         int supports_nonempty;
 416         int msg_buf_done = 0;
 417
 418         struct xilly_alloc_state rd_alloc = {
 419                 .salami = NULL,
 420                 .left_of_salami = 0,
 421                 .nbuffer = 1,
 422                 .direction = DMA_TO_DEVICE,
 423                 .regdirection = 0,
 424         };
 425
 426         struct xilly_alloc_state wr_alloc = {
 427                 .salami = NULL,
 428                 .left_of_salami = 0,
 429                 .nbuffer = 1,
 430                 .direction = DMA_FROM_DEVICE,
 431                 .regdirection = 0x80000000,
 432         };
 433
 434         channel = devm_kcalloc(dev, ep->num_channels,
 435                                sizeof(struct xilly_channel), GFP_KERNEL);
 436         if (!channel)
 437                 return -ENOMEM;
 438
 439         ep->channels = devm_kcalloc(dev, ep->num_channels + 1,
 440                                     sizeof(struct xilly_channel *),
 441                                     GFP_KERNEL);
 442         if (!ep->channels)
 443                 return -ENOMEM;
 444
 445         ep->channels[0] = NULL; /* Channel 0 is message buf. */
 446
 447         /* Initialize all channels with defaults */
 448
 449         for (i = 1; i <= ep->num_channels; i++) {
 450                 channel->wr_buffers = NULL;
 451                 channel->rd_buffers = NULL;
 452                 channel->num_wr_buffers = 0;
 453                 channel->num_rd_buffers = 0;
 454                 channel->wr_fpga_buf_idx = -1;
 455                 channel->wr_host_buf_idx = 0;
 456                 channel->wr_host_buf_pos = 0;
 457                 channel->wr_empty = 1;
 458                 channel->wr_ready = 0;
 459                 channel->wr_sleepy = 1;
 460                 channel->rd_fpga_buf_idx = 0;
 461                 channel->rd_host_buf_idx = 0;
 462                 channel->rd_host_buf_pos = 0;
 463                 channel->rd_full = 0;
 464                 channel->wr_ref_count = 0;
 465                 channel->rd_ref_count = 0;
 466
 467                 spin_lock_init(&channel->wr_spinlock);
 468                 spin_lock_init(&channel->rd_spinlock);
 469                 mutex_init(&channel->wr_mutex);
 470                 mutex_init(&channel->rd_mutex);
 471                 init_waitqueue_head(&channel->rd_wait);
 472                 init_waitqueue_head(&channel->wr_wait);
 473                 init_waitqueue_head(&channel->wr_ready_wait);
 474
 475                 INIT_DELAYED_WORK(&channel->rd_workitem, xillybus_autoflush);
 476
 477                 channel->endpoint = ep;
 478                 channel->chan_num = i;
 479
 480                 channel->log2_element_size = 0;
 481
 482                 ep->channels[i] = channel++;
 483         }
 484
 485         for (entry = 0; entry < entries; entry++, chandesc += 4) {
 486                 struct xilly_buffer **buffers = NULL;
 487
 488                 is_writebuf = chandesc[0] & 0x01;
 489                 channelnum = (chandesc[0] >> 1) | ((chandesc[1] & 0x0f) << 7);
 490                 format = (chandesc[1] >> 4) & 0x03;
 491                 allowpartial = (chandesc[1] >> 6) & 0x01;
 492                 synchronous = (chandesc[1] >> 7) & 0x01;
 493                 bufsize = 1 << (chandesc[2] & 0x1f);
 494                 bufnum = 1 << (chandesc[3] & 0x0f);
 495                 exclusive_open = (chandesc[2] >> 7) & 0x01;
 496                 seekable = (chandesc[2] >> 6) & 0x01;
 497                 supports_nonempty = (chandesc[2] >> 5) & 0x01;
 498
 499                 if ((channelnum > ep->num_channels) ||
 500                     ((channelnum == 0) && !is_writebuf)) {
 501                         dev_err(ep->dev,
 502                                 "IDT requests channel out of range. Aborting.\n");
 503                         return -ENODEV;
 504                 }
 505
 506                 channel = ep->channels[channelnum]; /* NULL for msg channel */
 507
 508                 if (!is_writebuf || channelnum > 0) {
 509                         channel->log2_element_size = ((format > 2) ?
 510                                                       2 : format);
 511
 512                         bytebufsize = channel->rd_buf_size = bufsize *
 513                                 (1 << channel->log2_element_size);
 514
 515                         buffers = devm_kcalloc(dev, bufnum,
 516                                                sizeof(struct xilly_buffer *),
 517                                                GFP_KERNEL);
 518                         if (!buffers)
 519                                 return -ENOMEM;
 520                 } else {
 521                         bytebufsize = bufsize << 2;
 522                 }
 523
 524                 if (!is_writebuf) {
 525                         channel->num_rd_buffers = bufnum;
 526                         channel->rd_allow_partial = allowpartial;
 527                         channel->rd_synchronous = synchronous;
 528                         channel->rd_exclusive_open = exclusive_open;
 529                         channel->seekable = seekable;
 530
 531                         channel->rd_buffers = buffers;
 532                         rc = xilly_get_dma_buffers(ep, &rd_alloc, buffers,
 533                                                    bufnum, bytebufsize);
 534                 } else if (channelnum > 0) {
 535                         channel->num_wr_buffers = bufnum;
 536
 537                         channel->seekable = seekable;
 538                         channel->wr_supports_nonempty = supports_nonempty;
 539
 540                         channel->wr_allow_partial = allowpartial;
 541                         channel->wr_synchronous = synchronous;
 542                         channel->wr_exclusive_open = exclusive_open;
 543
 544                         channel->wr_buffers = buffers;
 545                         rc = xilly_get_dma_buffers(ep, &wr_alloc, buffers,
 546                                                    bufnum, bytebufsize);
 547                 } else {
 548                         rc = xilly_get_dma_buffers(ep, &wr_alloc, NULL,
 549                                                    bufnum, bytebufsize);
 550                         msg_buf_done++;
 551                 }
 552
 553                 if (rc)
 554                         return -ENOMEM;
 555         }
 556
 557         if (!msg_buf_done) {
 558                 dev_err(ep->dev,
 559                         "Corrupt IDT: No message buffer. Aborting.\n");
 560                 return -ENODEV;
 561         }
 562         return 0;
 563 }
 564
 565 static int xilly_scan_idt(struct xilly_endpoint *endpoint,
 566                           struct xilly_idt_handle *idt_handle)
 567 {
 568         int count = 0;
 569         unsigned char *idt = endpoint->channels[1]->wr_buffers[0]->addr;
 570         unsigned char *end_of_idt = idt + endpoint->idtlen - 4;
 571         unsigned char *scan;
 572         int len;
 573
 574         scan = idt;
 575         idt_handle->idt = idt;
 576
 577         scan++; /* Skip version number */
 578
 579         while ((scan <= end_of_idt) && *scan) {
 580                 while ((scan <= end_of_idt) && *scan++)
 581                         /* Do nothing, just scan thru string */;
 582                 count++;
 583         }
 584
 585         scan++;
 586
 587         if (scan > end_of_idt) {
 588                 dev_err(endpoint->dev,
 589                         "IDT device name list overflow. Aborting.\n");
 590                 return -ENODEV;
 591         }
 592         idt_handle->chandesc = scan;
 593
 594         len = endpoint->idtlen - (3 + ((int) (scan - idt)));
 595
 596         if (len & 0x03) {
 597                 dev_err(endpoint->dev,
 598                         "Corrupt IDT device name list. Aborting.\n");
 599                 return -ENODEV;
 600         }
 601
 602         idt_handle->entries = len >> 2;
 603         endpoint->num_channels = count;
 604
 605         return 0;
 606 }
 607
 608 static int xilly_obtain_idt(struct xilly_endpoint *endpoint)
 609 {
 610         struct xilly_channel *channel;
 611         unsigned char *version;
 612         long t;
 613
 614         channel = endpoint->channels[1]; /* This should be generated ad-hoc */
 615
 616         channel->wr_sleepy = 1;
 617
 618         iowrite32(1 |
 619                   (3 << 24), /* Opcode 3 for channel 0 = Send IDT */
 620                   endpoint->registers + fpga_buf_ctrl_reg);
 621
 622         t = wait_event_interruptible_timeout(channel->wr_wait,
 623                                              (!channel->wr_sleepy),
 624                                              XILLY_TIMEOUT);
 625
 626         if (t <= 0) {
 627                 dev_err(endpoint->dev, "Failed to obtain IDT. Aborting.\n");
 628
 629                 if (endpoint->fatal_error)
 630                         return -EIO;
 631
 632                 return -ENODEV;
 633         }
 634
 635         endpoint->ephw->hw_sync_sgl_for_cpu(
 636                 channel->endpoint,
 637                 channel->wr_buffers[0]->dma_addr,
 638                 channel->wr_buf_size,
 639                 DMA_FROM_DEVICE);
 640
 641         if (channel->wr_buffers[0]->end_offset != endpoint->idtlen) {
 642                 dev_err(endpoint->dev,
 643                         "IDT length mismatch (%d != %d). Aborting.\n",
 644                         channel->wr_buffers[0]->end_offset, endpoint->idtlen);
 645                 return -ENODEV;
 646         }
 647
 648         if (crc32_le(~0, channel->wr_buffers[0]->addr,
 649                      endpoint->idtlen+1) != 0) {
 650                 dev_err(endpoint->dev, "IDT failed CRC check. Aborting.\n");
 651                 return -ENODEV;
 652         }
 653
 654         version = channel->wr_buffers[0]->addr;
 655
 656         /* Check version number. Accept anything below 0x82 for now. */
 657         if (*version > 0x82) {
 658                 dev_err(endpoint->dev,
 659                         "No support for IDT version 0x%02x. Maybe the xillybus driver needs an upgarde. Aborting.\n",
 660                         *version);
 661                 return -ENODEV;
 662         }
 663
 664         return 0;
 665 }
 666
 667 static ssize_t xillybus_read(struct file *filp, char __user *userbuf,
 668                              size_t count, loff_t *f_pos)
 669 {
 670         ssize_t rc;
 671         unsigned long flags;
 672         int bytes_done = 0;
 673         int no_time_left = 0;
 674         long deadline, left_to_sleep;
 675         struct xilly_channel *channel = filp->private_data;
 676
 677         int empty, reached_eof, exhausted, ready;
 678         /* Initializations are there only to silence warnings */
 679
 680         int howmany = 0, bufpos = 0, bufidx = 0, bufferdone = 0;
 681         int waiting_bufidx;
 682
 683         if (channel->endpoint->fatal_error)
 684                 return -EIO;
 685
 686         deadline = jiffies + 1 + XILLY_RX_TIMEOUT;
 687
 688         rc = mutex_lock_interruptible(&channel->wr_mutex);
 689         if (rc)
 690                 return rc;
 691
 692         while (1) { /* Note that we may drop mutex within this loop */
 693                 int bytes_to_do = count - bytes_done;
 694
 695                 spin_lock_irqsave(&channel->wr_spinlock, flags);
 696
 697                 empty = channel->wr_empty;
 698                 ready = !empty || channel->wr_ready;
 699
 700                 if (!empty) {
 701                         bufidx = channel->wr_host_buf_idx;
 702                         bufpos = channel->wr_host_buf_pos;
 703                         howmany = ((channel->wr_buffers[bufidx]->end_offset
 704                                     + 1) << channel->log2_element_size)
 705                                 - bufpos;
 706
 707                         /* Update wr_host_* to its post-operation state */
 708                         if (howmany > bytes_to_do) {
 709                                 bufferdone = 0;
 710
 711                                 howmany = bytes_to_do;
 712                                 channel->wr_host_buf_pos += howmany;
 713                         } else {
 714                                 bufferdone = 1;
 715
 716                                 channel->wr_host_buf_pos = 0;
 717
 718                                 if (bufidx == channel->wr_fpga_buf_idx) {
 719                                         channel->wr_empty = 1;
 720                                         channel->wr_sleepy = 1;
 721                                         channel->wr_ready = 0;
 722                                 }
 723
 724                                 if (bufidx >= (channel->num_wr_buffers - 1))
 725                                         channel->wr_host_buf_idx = 0;
 726                                 else
 727                                         channel->wr_host_buf_idx++;
 728                         }
 729                 }
 730
 731                 /*
 732                  * Marking our situation after the possible changes above,
 733                  * for use after releasing the spinlock.
 734                  *
 735                  * empty = empty before change
 736                  * exhasted = empty after possible change
 737                  */
 738
 739                 reached_eof = channel->wr_empty &&
 740                         (channel->wr_host_buf_idx == channel->wr_eof);
 741                 channel->wr_hangup = reached_eof;
 742                 exhausted = channel->wr_empty;
 743                 waiting_bufidx = channel->wr_host_buf_idx;
 744
 745                 spin_unlock_irqrestore(&channel->wr_spinlock, flags);
 746
 747                 if (!empty) { /* Go on, now without the spinlock */
 748
 749                         if (bufpos == 0) /* Position zero means it's virgin */
 750                                 channel->endpoint->ephw->hw_sync_sgl_for_cpu(
 751                                         channel->endpoint,
 752                                         channel->wr_buffers[bufidx]->dma_addr,
 753                                         channel->wr_buf_size,
 754                                         DMA_FROM_DEVICE);
 755
 756                         if (copy_to_user(
 757                                     userbuf,
 758                                     channel->wr_buffers[bufidx]->addr
 759                                     + bufpos, howmany))
 760                                 rc = -EFAULT;
 761
 762                         userbuf += howmany;
 763                         bytes_done += howmany;
 764
 765                         if (bufferdone) {
 766                                 channel->endpoint->ephw->hw_sync_sgl_for_device(
 767                                         channel->endpoint,
 768                                         channel->wr_buffers[bufidx]->dma_addr,
 769                                         channel->wr_buf_size,
 770                                         DMA_FROM_DEVICE);
 771
 772                                 /*
 773                                  * Tell FPGA the buffer is done with. It's an
 774                                  * atomic operation to the FPGA, so what
 775                                  * happens with other channels doesn't matter,
 776                                  * and the certain channel is protected with
 777                                  * the channel-specific mutex.
 778                                  */
 779
 780                                 iowrite32(1 | (channel->chan_num << 1) |
 781                                           (bufidx << 12),
 782                                           channel->endpoint->registers +
 783                                           fpga_buf_ctrl_reg);
 784                         }
 785
 786                         if (rc) {
 787                                 mutex_unlock(&channel->wr_mutex);
 788                                 return rc;
 789                         }
 790                 }
 791
 792                 /* This includes a zero-count return = EOF */
 793                 if ((bytes_done >= count) || reached_eof)
 794                         break;
 795
 796                 if (!exhausted)
 797                         continue; /* More in RAM buffer(s)? Just go on. */
 798
 799                 if ((bytes_done > 0) &&
 800                     (no_time_left ||
 801                      (channel->wr_synchronous && channel->wr_allow_partial)))
 802                         break;
 803
 804                 /*
 805                  * Nonblocking read: The "ready" flag tells us that the FPGA
 806                  * has data to send. In non-blocking mode, if it isn't on,
 807                  * just return. But if there is, we jump directly to the point
 808                  * where we ask for the FPGA to send all it has, and wait
 809                  * until that data arrives. So in a sense, we *do* block in
 810                  * nonblocking mode, but only for a very short time.
 811                  */
 812
 813                 if (!no_time_left && (filp->f_flags & O_NONBLOCK)) {
 814                         if (bytes_done > 0)
 815                                 break;
 816
 817                         if (ready)
 818                                 goto desperate;
 819
 820                         rc = -EAGAIN;
 821                         break;
 822                 }
 823
 824                 if (!no_time_left || (bytes_done > 0)) {
 825                         /*
 826                          * Note that in case of an element-misaligned read
 827                          * request, offsetlimit will include the last element,
 828                          * which will be partially read from.
 829                          */
 830                         int offsetlimit = ((count - bytes_done) - 1) >>
 831                                 channel->log2_element_size;
 832                         int buf_elements = channel->wr_buf_size >>
 833                                 channel->log2_element_size;
 834
 835                         /*
 836                          * In synchronous mode, always send an offset limit.
 837                          * Just don't send a value too big.
 838                          */
 839
 840                         if (channel->wr_synchronous) {
 841                                 /* Don't request more than one buffer */
 842                                 if (channel->wr_allow_partial &&
 843                                     (offsetlimit >= buf_elements))
 844                                         offsetlimit = buf_elements - 1;
 845
 846                                 /* Don't request more than all buffers */
 847                                 if (!channel->wr_allow_partial &&
 848                                     (offsetlimit >=
 849                                      (buf_elements * channel->num_wr_buffers)))
 850                                         offsetlimit = buf_elements *
 851                                                 channel->num_wr_buffers - 1;
 852                         }
 853
 854                         /*
 855                          * In asynchronous mode, force early flush of a buffer
 856                          * only if that will allow returning a full count. The
 857                          * "offsetlimit < ( ... )" rather than "<=" excludes
 858                          * requesting a full buffer, which would obviously
 859                          * cause a buffer transmission anyhow
 860                          */
 861
 862                         if (channel->wr_synchronous ||
 863                             (offsetlimit < (buf_elements - 1))) {
 864                                 mutex_lock(&channel->endpoint->register_mutex);
 865
 866                                 iowrite32(offsetlimit,
 867                                           channel->endpoint->registers +
 868                                           fpga_buf_offset_reg);
 869
 870                                 iowrite32(1 | (channel->chan_num << 1) |
 871                                           (2 << 24) |  /* 2 = offset limit */
 872                                           (waiting_bufidx << 12),
 873                                           channel->endpoint->registers +
 874                                           fpga_buf_ctrl_reg);
 875
 876                                 mutex_unlock(&channel->endpoint->
 877                                              register_mutex);
 878                         }
 879                 }
 880
 881                 /*
 882                  * If partial completion is disallowed, there is no point in
 883                  * timeout sleeping. Neither if no_time_left is set and
 884                  * there's no data.
 885                  */
 886
 887                 if (!channel->wr_allow_partial ||
 888                     (no_time_left && (bytes_done == 0))) {
 889                         /*
 890                          * This do-loop will run more than once if another
 891                          * thread reasserted wr_sleepy before we got the mutex
 892                          * back, so we try again.
 893                          */
 894
 895                         do {
 896                                 mutex_unlock(&channel->wr_mutex);
 897
 898                                 if (wait_event_interruptible(
 899                                             channel->wr_wait,
 900                                             (!channel->wr_sleepy)))
 901                                         goto interrupted;
 902
 903                                 if (mutex_lock_interruptible(
 904                                             &channel->wr_mutex))
 905                                         goto interrupted;
 906                         } while (channel->wr_sleepy);
 907
 908                         continue;
 909
 910 interrupted: /* Mutex is not held if got here */
 911                         if (channel->endpoint->fatal_error)
 912                                 return -EIO;
 913                         if (bytes_done)
 914                                 return bytes_done;
 915                         if (filp->f_flags & O_NONBLOCK)
 916                                 return -EAGAIN; /* Don't admit snoozing */
 917                         return -EINTR;
 918                 }
 919
 920                 left_to_sleep = deadline - ((long) jiffies);
 921
 922                 /*
 923                  * If our time is out, skip the waiting. We may miss wr_sleepy
 924                  * being deasserted but hey, almost missing the train is like
 925                  * missing it.
 926                  */
 927
 928                 if (left_to_sleep > 0) {
 929                         left_to_sleep =
 930                                 wait_event_interruptible_timeout(
 931                                         channel->wr_wait,
 932                                         (!channel->wr_sleepy),
 933                                         left_to_sleep);
 934
 935                         if (left_to_sleep > 0) /* wr_sleepy deasserted */
 936                                 continue;
 937
 938                         if (left_to_sleep < 0) { /* Interrupt */
 939                                 mutex_unlock(&channel->wr_mutex);
 940                                 if (channel->endpoint->fatal_error)
 941                                         return -EIO;
 942                                 if (bytes_done)
 943                                         return bytes_done;
 944                                 return -EINTR;
 945                         }
 946                 }
 947
 948 desperate:
 949                 no_time_left = 1; /* We're out of sleeping time. Desperate! */
 950
 951                 if (bytes_done == 0) {
 952                         /*
 953                          * Reaching here means that we allow partial return,
 954                          * that we've run out of time, and that we have
 955                          * nothing to return.
 956                          * So tell the FPGA to send anything it has or gets.
 957                          */
 958
 959                         iowrite32(1 | (channel->chan_num << 1) |
 960                                   (3 << 24) |  /* Opcode 3, flush it all! */
 961                                   (waiting_bufidx << 12),
 962                                   channel->endpoint->registers +
 963                                   fpga_buf_ctrl_reg);
 964                 }
 965
 966                 /*
 967                  * Reaching here means that we *do* have data in the buffer,
 968                  * but the "partial" flag disallows returning less than
 969                  * required. And we don't have as much. So loop again,
 970                  * which is likely to end up blocking indefinitely until
 971                  * enough data has arrived.
 972                  */
 973         }
 974
 975         mutex_unlock(&channel->wr_mutex);
 976
 977         if (channel->endpoint->fatal_error)
 978                 return -EIO;
 979
 980         if (rc)
 981                 return rc;
 982
 983         return bytes_done;
 984 }
 985
 986 /*
 987  * The timeout argument takes values as follows:
 988  *  >0 : Flush with timeout
 989  * ==0 : Flush, and wait idefinitely for the flush to complete
 990  *  <0 : Autoflush: Flush only if there's a single buffer occupied
 991  */
 992
 993 static int xillybus_myflush(struct xilly_channel *channel, long timeout)
 994 {
 995         int rc;
 996         unsigned long flags;
 997
 998         int end_offset_plus1;
 999         int bufidx, bufidx_minus1;
1000         int i;
1001         int empty;
1002         int new_rd_host_buf_pos;
1003
1004         if (channel->endpoint->fatal_error)
1005                 return -EIO;
1006         rc = mutex_lock_interruptible(&channel->rd_mutex);
1007         if (rc)
1008                 return rc;
1009
1010         /*
1011          * Don't flush a closed channel. This can happen when the work queued
1012          * autoflush thread fires off after the file has closed. This is not
1013          * an error, just something to dismiss.
1014          */
1015
1016         if (!channel->rd_ref_count)
1017                 goto done;
1018
1019         bufidx = channel->rd_host_buf_idx;
1020
1021         bufidx_minus1 = (bufidx == 0) ?
1022                 channel->num_rd_buffers - 1 :
1023                 bufidx - 1;
1024
1025         end_offset_plus1 = channel->rd_host_buf_pos >>
1026                 channel->log2_element_size;
1027
1028         new_rd_host_buf_pos = channel->rd_host_buf_pos -
1029                 (end_offset_plus1 << channel->log2_element_size);
1030
1031         /* Submit the current buffer if it's nonempty */
1032         if (end_offset_plus1) {
1033                 unsigned char *tail = channel->rd_buffers[bufidx]->addr +
1034                         (end_offset_plus1 << channel->log2_element_size);
1035
1036                 /* Copy  unflushed data, so we can put it in next buffer */
1037                 for (i = 0; i < new_rd_host_buf_pos; i++)
1038                         channel->rd_leftovers[i] = *tail++;
1039
1040                 spin_lock_irqsave(&channel->rd_spinlock, flags);
1041
1042                 /* Autoflush only if a single buffer is occupied */
1043
1044                 if ((timeout < 0) &&
1045                     (channel->rd_full ||
1046                      (bufidx_minus1 != channel->rd_fpga_buf_idx))) {
1047                         spin_unlock_irqrestore(&channel->rd_spinlock, flags);
1048                         /*
1049                          * A new work item may be queued by the ISR exactly
1050                          * now, since the execution of a work item allows the
1051                          * queuing of a new one while it's running.
1052                          */
1053                         goto done;
1054                 }
1055
1056                 /* The 4th element is never needed for data, so it's a flag */
1057                 channel->rd_leftovers[3] = (new_rd_host_buf_pos != 0);
1058
1059                 /* Set up rd_full to reflect a certain moment's state */
1060
1061                 if (bufidx == channel->rd_fpga_buf_idx)
1062                         channel->rd_full = 1;
1063                 spin_unlock_irqrestore(&channel->rd_spinlock, flags);
1064
1065                 if (bufidx >= (channel->num_rd_buffers - 1))
1066                         channel->rd_host_buf_idx = 0;
1067                 else
1068                         channel->rd_host_buf_idx++;
1069
1070                 channel->endpoint->ephw->hw_sync_sgl_for_device(
1071                         channel->endpoint,
1072                         channel->rd_buffers[bufidx]->dma_addr,
1073                         channel->rd_buf_size,
1074                         DMA_TO_DEVICE);
1075
1076                 mutex_lock(&channel->endpoint->register_mutex);
1077
1078                 iowrite32(end_offset_plus1 - 1,
1079                           channel->endpoint->registers + fpga_buf_offset_reg);
1080
1081                 iowrite32((channel->chan_num << 1) | /* Channel ID */
1082                           (2 << 24) |  /* Opcode 2, submit buffer */
1083                           (bufidx << 12),
1084                           channel->endpoint->registers + fpga_buf_ctrl_reg);
1085
1086                 mutex_unlock(&channel->endpoint->register_mutex);
1087         } else if (bufidx == 0) {
1088                 bufidx = channel->num_rd_buffers - 1;
1089         } else {
1090                 bufidx--;
1091         }
1092
1093         channel->rd_host_buf_pos = new_rd_host_buf_pos;
1094
1095         if (timeout < 0)
1096                 goto done; /* Autoflush */
1097
1098         /*
1099          * bufidx is now the last buffer written to (or equal to
1100          * rd_fpga_buf_idx if buffer was never written to), and
1101          * channel->rd_host_buf_idx the one after it.
1102          *
1103          * If bufidx == channel->rd_fpga_buf_idx we're either empty or full.
1104          */
1105
1106         while (1) { /* Loop waiting for draining of buffers */
1107                 spin_lock_irqsave(&channel->rd_spinlock, flags);
1108
1109                 if (bufidx != channel->rd_fpga_buf_idx)
1110                         channel->rd_full = 1; /*
1111                                                * Not really full,
1112                                                * but needs waiting.
1113                                                */
1114
1115                 empty = !channel->rd_full;
1116
1117                 spin_unlock_irqrestore(&channel->rd_spinlock, flags);
1118
1119                 if (empty)
1120                         break;
1121
1122                 /*
1123                  * Indefinite sleep with mutex taken. With data waiting for
1124                  * flushing user should not be surprised if open() for write
1125                  * sleeps.
1126                  */
1127                 if (timeout == 0)
1128                         wait_event_interruptible(channel->rd_wait,
1129                                                  (!channel->rd_full));
1130
1131                 else if (wait_event_interruptible_timeout(
1132                                  channel->rd_wait,
1133                                  (!channel->rd_full),
1134                                  timeout) == 0) {
1135                         dev_warn(channel->endpoint->dev,
1136                                  "Timed out while flushing. Output data may be lost.\n");
1137
1138                         rc = -ETIMEDOUT;
1139                         break;
1140                 }
1141
1142                 if (channel->rd_full) {
1143                         rc = -EINTR;
1144                         break;
1145                 }
1146         }
1147
1148 done:
1149         mutex_unlock(&channel->rd_mutex);
1150
1151         if (channel->endpoint->fatal_error)
1152                 return -EIO;
1153
1154         return rc;
1155 }
1156
1157 static int xillybus_flush(struct file *filp, fl_owner_t id)
1158 {
1159         if (!(filp->f_mode & FMODE_WRITE))
1160                 return 0;
1161
1162         return xillybus_myflush(filp->private_data, HZ); /* 1 second timeout */
1163 }
1164
1165 static void xillybus_autoflush(struct work_struct *work)
1166 {
1167         struct delayed_work *workitem = container_of(
1168                 work, struct delayed_work, work);
1169         struct xilly_channel *channel = container_of(
1170                 workitem, struct xilly_channel, rd_workitem);
1171         int rc;
1172
1173         rc = xillybus_myflush(channel, -1);
1174         if (rc == -EINTR)
1175                 dev_warn(channel->endpoint->dev,
1176                          "Autoflush failed because work queue thread got a signal.\n");
1177         else if (rc)
1178                 dev_err(channel->endpoint->dev,
1179                         "Autoflush failed under weird circumstances.\n");
1180 }
1181
1182 static ssize_t xillybus_write(struct file *filp, const char __user *userbuf,
1183                               size_t count, loff_t *f_pos)
1184 {
1185         ssize_t rc;
1186         unsigned long flags;
1187         int bytes_done = 0;
1188         struct xilly_channel *channel = filp->private_data;
1189
1190         int full, exhausted;
1191         /* Initializations are there only to silence warnings */
1192
1193         int howmany = 0, bufpos = 0, bufidx = 0, bufferdone = 0;
1194         int end_offset_plus1 = 0;
1195
1196         if (channel->endpoint->fatal_error)
1197                 return -EIO;
1198
1199         rc = mutex_lock_interruptible(&channel->rd_mutex);
1200         if (rc)
1201                 return rc;
1202
1203         while (1) {
1204                 int bytes_to_do = count - bytes_done;
1205
1206                 spin_lock_irqsave(&channel->rd_spinlock, flags);
1207
1208                 full = channel->rd_full;
1209
1210                 if (!full) {
1211                         bufidx = channel->rd_host_buf_idx;
1212                         bufpos = channel->rd_host_buf_pos;
1213                         howmany = channel->rd_buf_size - bufpos;
1214
1215                         /*
1216                          * Update rd_host_* to its state after this operation.
1217                          * count=0 means committing the buffer immediately,
1218                          * which is like flushing, but not necessarily block.
1219                          */
1220
1221                         if ((howmany > bytes_to_do) &&
1222                             (count ||
1223                              ((bufpos >> channel->log2_element_size) == 0))) {
1224                                 bufferdone = 0;
1225
1226                                 howmany = bytes_to_do;
1227                                 channel->rd_host_buf_pos += howmany;
1228                         } else {
1229                                 bufferdone = 1;
1230
1231                                 if (count) {
1232                                         end_offset_plus1 =
1233                                                 channel->rd_buf_size >>
1234                                                 channel->log2_element_size;
1235                                         channel->rd_host_buf_pos = 0;
1236                                 } else {
1237                                         unsigned char *tail;
1238                                         int i;
1239
1240                                         end_offset_plus1 = bufpos >>
1241                                                 channel->log2_element_size;
1242
1243                                         channel->rd_host_buf_pos -=
1244                                                 end_offset_plus1 <<
1245                                                 channel->log2_element_size;
1246
1247                                         tail = channel->
1248                                                 rd_buffers[bufidx]->addr +
1249                                                 (end_offset_plus1 <<
1250                                                  channel->log2_element_size);
1251
1252                                         for (i = 0;
1253                                              i < channel->rd_host_buf_pos;
1254                                              i++)
1255                                                 channel->rd_leftovers[i] =
1256                                                         *tail++;
1257                                 }
1258
1259                                 if (bufidx == channel->rd_fpga_buf_idx)
1260                                         channel->rd_full = 1;
1261
1262                                 if (bufidx >= (channel->num_rd_buffers - 1))
1263                                         channel->rd_host_buf_idx = 0;
1264                                 else
1265                                         channel->rd_host_buf_idx++;
1266                         }
1267                 }
1268
1269                 /*
1270                  * Marking our situation after the possible changes above,
1271                  * for use  after releasing the spinlock.
1272                  *
1273                  * full = full before change
1274                  * exhasted = full after possible change
1275                  */
1276
1277                 exhausted = channel->rd_full;
1278
1279                 spin_unlock_irqrestore(&channel->rd_spinlock, flags);
1280
1281                 if (!full) { /* Go on, now without the spinlock */
1282                         unsigned char *head =
1283                                 channel->rd_buffers[bufidx]->addr;
1284                         int i;
1285
1286                         if ((bufpos == 0) || /* Zero means it's virgin */
1287                             (channel->rd_leftovers[3] != 0)) {
1288                                 channel->endpoint->ephw->hw_sync_sgl_for_cpu(
1289                                         channel->endpoint,
1290                                         channel->rd_buffers[bufidx]->dma_addr,
1291                                         channel->rd_buf_size,
1292                                         DMA_TO_DEVICE);
1293
1294                                 /* Virgin, but leftovers are due */
1295                                 for (i = 0; i < bufpos; i++)
1296                                         *head++ = channel->rd_leftovers[i];
1297
1298                                 channel->rd_leftovers[3] = 0; /* Clear flag */
1299                         }
1300
1301                         if (copy_from_user(
1302                                     channel->rd_buffers[bufidx]->addr + bufpos,
1303                                     userbuf, howmany))
1304                                 rc = -EFAULT;
1305
1306                         userbuf += howmany;
1307                         bytes_done += howmany;
1308
1309                         if (bufferdone) {
1310                                 channel->endpoint->ephw->hw_sync_sgl_for_device(
1311                                         channel->endpoint,
1312                                         channel->rd_buffers[bufidx]->dma_addr,
1313                                         channel->rd_buf_size,
1314                                         DMA_TO_DEVICE);
1315
1316                                 mutex_lock(&channel->endpoint->register_mutex);
1317
1318                                 iowrite32(end_offset_plus1 - 1,
1319                                           channel->endpoint->registers +
1320                                           fpga_buf_offset_reg);
1321
1322                                 iowrite32((channel->chan_num << 1) |
1323                                           (2 << 24) |  /* 2 = submit buffer */
1324                                           (bufidx << 12),
1325                                           channel->endpoint->registers +
1326                                           fpga_buf_ctrl_reg);
1327
1328                                 mutex_unlock(&channel->endpoint->
1329                                              register_mutex);
1330
1331                                 channel->rd_leftovers[3] =
1332                                         (channel->rd_host_buf_pos != 0);
1333                         }
1334
1335                         if (rc) {
1336                                 mutex_unlock(&channel->rd_mutex);
1337
1338                                 if (channel->endpoint->fatal_error)
1339                                         return -EIO;
1340
1341                                 if (!channel->rd_synchronous)
1342                                         queue_delayed_work(
1343                                                 xillybus_wq,
1344                                                 &channel->rd_workitem,
1345                                                 XILLY_RX_TIMEOUT);
1346
1347                                 return rc;
1348                         }
1349                 }
1350
1351                 if (bytes_done >= count)
1352                         break;
1353
1354                 if (!exhausted)
1355                         continue; /* If there's more space, just go on */
1356
1357                 if ((bytes_done > 0) && channel->rd_allow_partial)
1358                         break;
1359
1360                 /*
1361                  * Indefinite sleep with mutex taken. With data waiting for
1362                  * flushing, user should not be surprised if open() for write
1363                  * sleeps.
1364                  */
1365
1366                 if (filp->f_flags & O_NONBLOCK) {
1367                         rc = -EAGAIN;
1368                         break;
1369                 }
1370
1371                 if (wait_event_interruptible(channel->rd_wait,
1372                                              (!channel->rd_full))) {
1373                         mutex_unlock(&channel->rd_mutex);
1374
1375                         if (channel->endpoint->fatal_error)
1376                                 return -EIO;
1377
1378                         if (bytes_done)
1379                                 return bytes_done;
1380                         return -EINTR;
1381                 }
1382         }
1383
1384         mutex_unlock(&channel->rd_mutex);
1385
1386         if (!channel->rd_synchronous)
1387                 queue_delayed_work(xillybus_wq,
1388                                    &channel->rd_workitem,
1389                                    XILLY_RX_TIMEOUT);
1390
1391         if (channel->endpoint->fatal_error)
1392                 return -EIO;
1393
1394         if (rc)
1395                 return rc;
1396
1397         if ((channel->rd_synchronous) && (bytes_done > 0)) {
1398                 rc = xillybus_myflush(filp->private_data, 0); /* No timeout */
1399
1400                 if (rc && (rc != -EINTR))
1401                         return rc;
1402         }
1403
1404         return bytes_done;
1405 }
1406
1407 static int xillybus_open(struct inode *inode, struct file *filp)
1408 {
1409         int rc = 0;
1410         unsigned long flags;
1411         int minor = iminor(inode);
1412         int major = imajor(inode);
1413         struct xilly_endpoint *ep_iter, *endpoint = NULL;
1414         struct xilly_channel *channel;
1415
1416         mutex_lock(&ep_list_lock);
1417
1418         list_for_each_entry(ep_iter, &list_of_endpoints, ep_list) {
1419                 if ((ep_iter->major == major) &&
1420                     (minor >= ep_iter->lowest_minor) &&
1421                     (minor < (ep_iter->lowest_minor +
1422                               ep_iter->num_channels))) {
1423                         endpoint = ep_iter;
1424                         break;
1425                 }
1426         }
1427         mutex_unlock(&ep_list_lock);
1428
1429         if (!endpoint) {
1430                 pr_err("xillybus: open() failed to find a device for major=%d and minor=%d\n",
1431                        major, minor);
1432                 return -ENODEV;
1433         }
1434
1435         if (endpoint->fatal_error)
1436                 return -EIO;
1437
1438         channel = endpoint->channels[1 + minor - endpoint->lowest_minor];
1439         filp->private_data = channel;
1440
1441         /*
1442          * It gets complicated because:
1443          * 1. We don't want to take a mutex we don't have to
1444          * 2. We don't want to open one direction if the other will fail.
1445          */
1446
1447         if ((filp->f_mode & FMODE_READ) && (!channel->num_wr_buffers))
1448                 return -ENODEV;
1449
1450         if ((filp->f_mode & FMODE_WRITE) && (!channel->num_rd_buffers))
1451                 return -ENODEV;
1452
1453         if ((filp->f_mode & FMODE_READ) && (filp->f_flags & O_NONBLOCK) &&
1454             (channel->wr_synchronous || !channel->wr_allow_partial ||
1455              !channel->wr_supports_nonempty)) {
1456                 dev_err(endpoint->dev,
1457                         "open() failed: O_NONBLOCK not allowed for read on this device\n");
1458                 return -ENODEV;
1459         }
1460
1461         if ((filp->f_mode & FMODE_WRITE) && (filp->f_flags & O_NONBLOCK) &&
1462             (channel->rd_synchronous || !channel->rd_allow_partial)) {
1463                 dev_err(endpoint->dev,
1464                         "open() failed: O_NONBLOCK not allowed for write on this device\n");
1465                 return -ENODEV;
1466         }
1467
1468         /*
1469          * Note: open() may block on getting mutexes despite O_NONBLOCK.
1470          * This shouldn't occur normally, since multiple open of the same
1471          * file descriptor is almost always prohibited anyhow
1472          * (*_exclusive_open is normally set in real-life systems).
1473          */
1474
1475         if (filp->f_mode & FMODE_READ) {
1476                 rc = mutex_lock_interruptible(&channel->wr_mutex);
1477                 if (rc)
1478                         return rc;
1479         }
1480
1481         if (filp->f_mode & FMODE_WRITE) {
1482                 rc = mutex_lock_interruptible(&channel->rd_mutex);
1483                 if (rc)
1484                         goto unlock_wr;
1485         }
1486
1487         if ((filp->f_mode & FMODE_READ) &&
1488             (channel->wr_ref_count != 0) &&
1489             (channel->wr_exclusive_open)) {
1490                 rc = -EBUSY;
1491                 goto unlock;
1492         }
1493
1494         if ((filp->f_mode & FMODE_WRITE) &&
1495             (channel->rd_ref_count != 0) &&
1496             (channel->rd_exclusive_open)) {
1497                 rc = -EBUSY;
1498                 goto unlock;
1499         }
1500
1501         if (filp->f_mode & FMODE_READ) {
1502                 if (channel->wr_ref_count == 0) { /* First open of file */
1503                         /* Move the host to first buffer */
1504                         spin_lock_irqsave(&channel->wr_spinlock, flags);
1505                         channel->wr_host_buf_idx = 0;
1506                         channel->wr_host_buf_pos = 0;
1507                         channel->wr_fpga_buf_idx = -1;
1508                         channel->wr_empty = 1;
1509                         channel->wr_ready = 0;
1510                         channel->wr_sleepy = 1;
1511                         channel->wr_eof = -1;
1512                         channel->wr_hangup = 0;
1513
1514                         spin_unlock_irqrestore(&channel->wr_spinlock, flags);
1515
1516                         iowrite32(1 | (channel->chan_num << 1) |
1517                                   (4 << 24) |  /* Opcode 4, open channel */
1518                                   ((channel->wr_synchronous & 1) << 23),
1519                                   channel->endpoint->registers +
1520                                   fpga_buf_ctrl_reg);
1521                 }
1522
1523                 channel->wr_ref_count++;
1524         }
1525
1526         if (filp->f_mode & FMODE_WRITE) {
1527                 if (channel->rd_ref_count == 0) { /* First open of file */
1528                         /* Move the host to first buffer */
1529                         spin_lock_irqsave(&channel->rd_spinlock, flags);
1530                         channel->rd_host_buf_idx = 0;
1531                         channel->rd_host_buf_pos = 0;
1532                         channel->rd_leftovers[3] = 0; /* No leftovers. */
1533                         channel->rd_fpga_buf_idx = channel->num_rd_buffers - 1;
1534                         channel->rd_full = 0;
1535
1536                         spin_unlock_irqrestore(&channel->rd_spinlock, flags);
1537
1538                         iowrite32((channel->chan_num << 1) |
1539                                   (4 << 24),   /* Opcode 4, open channel */
1540                                   channel->endpoint->registers +
1541                                   fpga_buf_ctrl_reg);
1542                 }
1543
1544                 channel->rd_ref_count++;
1545         }
1546
1547 unlock:
1548         if (filp->f_mode & FMODE_WRITE)
1549                 mutex_unlock(&channel->rd_mutex);
1550 unlock_wr:
1551         if (filp->f_mode & FMODE_READ)
1552                 mutex_unlock(&channel->wr_mutex);
1553
1554         if (!rc && (!channel->seekable))
1555                 return nonseekable_open(inode, filp);
1556
1557         return rc;
1558 }
1559
1560 static int xillybus_release(struct inode *inode, struct file *filp)
1561 {
1562         unsigned long flags;
1563         struct xilly_channel *channel = filp->private_data;
1564
1565         int buf_idx;
1566         int eof;
1567
1568         if (channel->endpoint->fatal_error)
1569                 return -EIO;
1570
1571         if (filp->f_mode & FMODE_WRITE) {
1572                 mutex_lock(&channel->rd_mutex);
1573
1574                 channel->rd_ref_count--;
1575
1576                 if (channel->rd_ref_count == 0) {
1577                         /*
1578                          * We rely on the kernel calling flush()
1579                          * before we get here.
1580                          */
1581
1582                         iowrite32((channel->chan_num << 1) | /* Channel ID */
1583                                   (5 << 24),  /* Opcode 5, close channel */
1584                                   channel->endpoint->registers +
1585                                   fpga_buf_ctrl_reg);
1586                 }
1587                 mutex_unlock(&channel->rd_mutex);
1588         }
1589
1590         if (filp->f_mode & FMODE_READ) {
1591                 mutex_lock(&channel->wr_mutex);
1592
1593                 channel->wr_ref_count--;
1594
1595                 if (channel->wr_ref_count == 0) {
1596                         iowrite32(1 | (channel->chan_num << 1) |
1597                                   (5 << 24),  /* Opcode 5, close channel */
1598                                   channel->endpoint->registers +
1599                                   fpga_buf_ctrl_reg);
1600
1601                         /*
1602                          * This is crazily cautious: We make sure that not
1603                          * only that we got an EOF (be it because we closed
1604                          * the channel or because of a user's EOF), but verify
1605                          * that it's one beyond the last buffer arrived, so
1606                          * we have no leftover buffers pending before wrapping
1607                          * up (which can only happen in asynchronous channels,
1608                          * BTW)
1609                          */
1610
1611                         while (1) {
1612                                 spin_lock_irqsave(&channel->wr_spinlock,
1613                                                   flags);
1614                                 buf_idx = channel->wr_fpga_buf_idx;
1615                                 eof = channel->wr_eof;
1616                                 channel->wr_sleepy = 1;
1617                                 spin_unlock_irqrestore(&channel->wr_spinlock,
1618                                                        flags);
1619
1620                                 /*
1621                                  * Check if eof points at the buffer after
1622                                  * the last one the FPGA submitted. Note that
1623                                  * no EOF is marked by negative eof.
1624                                  */
1625
1626                                 buf_idx++;
1627                                 if (buf_idx == channel->num_wr_buffers)
1628                                         buf_idx = 0;
1629
1630                                 if (buf_idx == eof)
1631                                         break;
1632
1633                                 /*
1634                                  * Steal extra 100 ms if awaken by interrupt.
1635                                  * This is a simple workaround for an
1636                                  * interrupt pending when entering, which would
1637                                  * otherwise result in declaring the hardware
1638                                  * non-responsive.
1639                                  */
1640
1641                                 if (wait_event_interruptible(
1642                                             channel->wr_wait,
1643                                             (!channel->wr_sleepy)))
1644                                         msleep(100);
1645
1646                                 if (channel->wr_sleepy) {
1647                                         mutex_unlock(&channel->wr_mutex);
1648                                         dev_warn(channel->endpoint->dev,
1649                                                  "Hardware failed to respond to close command, therefore left in messy state.\n");
1650                                         return -EINTR;
1651                                 }
1652                         }
1653                 }
1654
1655                 mutex_unlock(&channel->wr_mutex);
1656         }
1657
1658         return 0;
1659 }
1660
1661 static loff_t xillybus_llseek(struct file *filp, loff_t offset, int whence)
1662 {
1663         struct xilly_channel *channel = filp->private_data;
1664         loff_t pos = filp->f_pos;
1665         int rc = 0;
1666
1667         /*
1668          * Take both mutexes not allowing interrupts, since it seems like
1669          * common applications don't expect an -EINTR here. Besides, multiple
1670          * access to a single file descriptor on seekable devices is a mess
1671          * anyhow.
1672          */
1673
1674         if (channel->endpoint->fatal_error)
1675                 return -EIO;
1676
1677         mutex_lock(&channel->wr_mutex);
1678         mutex_lock(&channel->rd_mutex);
1679
1680         switch (whence) {
1681         case SEEK_SET:
1682                 pos = offset;
1683                 break;
1684         case SEEK_CUR:
1685                 pos += offset;
1686                 break;
1687         case SEEK_END:
1688                 pos = offset; /* Going to the end => to the beginning */
1689                 break;
1690         default:
1691                 rc = -EINVAL;
1692                 goto end;
1693         }
1694
1695         /* In any case, we must finish on an element boundary */
1696         if (pos & ((1 << channel->log2_element_size) - 1)) {
1697                 rc = -EINVAL;
1698                 goto end;
1699         }
1700
1701         mutex_lock(&channel->endpoint->register_mutex);
1702
1703         iowrite32(pos >> channel->log2_element_size,
1704                   channel->endpoint->registers + fpga_buf_offset_reg);
1705
1706         iowrite32((channel->chan_num << 1) |
1707                   (6 << 24),  /* Opcode 6, set address */
1708                   channel->endpoint->registers + fpga_buf_ctrl_reg);
1709
1710         mutex_unlock(&channel->endpoint->register_mutex);
1711
1712 end:
1713         mutex_unlock(&channel->rd_mutex);
1714         mutex_unlock(&channel->wr_mutex);
1715
1716         if (rc) /* Return error after releasing mutexes */
1717                 return rc;
1718
1719         filp->f_pos = pos;
1720
1721         /*
1722          * Since seekable devices are allowed only when the channel is
1723          * synchronous, we assume that there is no data pending in either
1724          * direction (which holds true as long as no concurrent access on the
1725          * file descriptor takes place).
1726          * The only thing we may need to throw away is leftovers from partial
1727          * write() flush.
1728          */
1729
1730         channel->rd_leftovers[3] = 0;
1731
1732         return pos;
1733 }
1734
1735 static unsigned int xillybus_poll(struct file *filp, poll_table *wait)
1736 {
1737         struct xilly_channel *channel = filp->private_data;
1738         unsigned int mask = 0;
1739         unsigned long flags;
1740
1741         poll_wait(filp, &channel->endpoint->ep_wait, wait);
1742
1743         /*
1744          * poll() won't play ball regarding read() channels which
1745          * aren't asynchronous and support the nonempty message. Allowing
1746          * that will create situations where data has been delivered at
1747          * the FPGA, and users expecting select() to wake up, which it may
1748          * not.
1749          */
1750
1751         if (!channel->wr_synchronous && channel->wr_supports_nonempty) {
1752                 poll_wait(filp, &channel->wr_wait, wait);
1753                 poll_wait(filp, &channel->wr_ready_wait, wait);
1754
1755                 spin_lock_irqsave(&channel->wr_spinlock, flags);
1756                 if (!channel->wr_empty || channel->wr_ready)
1757                         mask |= POLLIN | POLLRDNORM;
1758
1759                 if (channel->wr_hangup)
1760                         /*
1761                          * Not POLLHUP, because its behavior is in the
1762                          * mist, and POLLIN does what we want: Wake up
1763                          * the read file descriptor so it sees EOF.
1764                          */
1765                         mask |=  POLLIN | POLLRDNORM;
1766                 spin_unlock_irqrestore(&channel->wr_spinlock, flags);
1767         }
1768
1769         /*
1770          * If partial data write is disallowed on a write() channel,
1771          * it's pointless to ever signal OK to write, because is could
1772          * block despite some space being available.
1773          */
1774
1775         if (channel->rd_allow_partial) {
1776                 poll_wait(filp, &channel->rd_wait, wait);
1777
1778                 spin_lock_irqsave(&channel->rd_spinlock, flags);
1779                 if (!channel->rd_full)
1780                         mask |= POLLOUT | POLLWRNORM;
1781                 spin_unlock_irqrestore(&channel->rd_spinlock, flags);
1782         }
1783
1784         if (channel->endpoint->fatal_error)
1785                 mask |= POLLERR;
1786
1787         return mask;
1788 }
1789
1790 static const struct file_operations xillybus_fops = {
1791         .owner      = THIS_MODULE,
1792         .read       = xillybus_read,
1793         .write      = xillybus_write,
1794         .open       = xillybus_open,
1795         .flush      = xillybus_flush,
1796         .release    = xillybus_release,
1797         .llseek     = xillybus_llseek,
1798         .poll       = xillybus_poll,
1799 };
1800
1801 static int xillybus_init_chrdev(struct xilly_endpoint *endpoint,
1802                                 const unsigned char *idt)
1803 {
1804         int rc;
1805         dev_t dev;
1806         int devnum, i, minor, major;
1807         char devname[48];
1808         struct device *device;
1809
1810         rc = alloc_chrdev_region(&dev, 0, /* minor start */
1811                                  endpoint->num_channels,
1812                                  xillyname);
1813         if (rc) {
1814                 dev_warn(endpoint->dev, "Failed to obtain major/minors");
1815                 return rc;
1816         }
1817
1818         endpoint->major = major = MAJOR(dev);
1819         endpoint->lowest_minor = minor = MINOR(dev);
1820
1821         cdev_init(&endpoint->cdev, &xillybus_fops);
1822         endpoint->cdev.owner = endpoint->ephw->owner;
1823         rc = cdev_add(&endpoint->cdev, MKDEV(major, minor),
1824                       endpoint->num_channels);
1825         if (rc) {
1826                 dev_warn(endpoint->dev, "Failed to add cdev. Aborting.\n");
1827                 goto unregister_chrdev;
1828         }
1829
1830         idt++;
1831
1832         for (i = minor, devnum = 0;
1833              devnum < endpoint->num_channels;
1834              devnum++, i++) {
1835                 snprintf(devname, sizeof(devname)-1, "xillybus_%s", idt);
1836
1837                 devname[sizeof(devname)-1] = 0; /* Should never matter */
1838
1839                 while (*idt++)
1840                         /* Skip to next */;
1841
1842                 device = device_create(xillybus_class,
1843                                        NULL,
1844                                        MKDEV(major, i),
1845                                        NULL,
1846                                        "%s", devname);
1847
1848                 if (IS_ERR(device)) {
1849                         dev_warn(endpoint->dev,
1850                                  "Failed to create %s device. Aborting.\n",
1851                                  devname);
1852                         rc = -ENODEV;
1853                         goto unroll_device_create;
1854                 }
1855         }
1856
1857         dev_info(endpoint->dev, "Created %d device files.\n",
1858                  endpoint->num_channels);
1859         return 0; /* succeed */
1860
1861 unroll_device_create:
1862         devnum--; i--;
1863         for (; devnum >= 0; devnum--, i--)
1864                 device_destroy(xillybus_class, MKDEV(major, i));
1865
1866         cdev_del(&endpoint->cdev);
1867 unregister_chrdev:
1868         unregister_chrdev_region(MKDEV(major, minor), endpoint->num_channels);
1869
1870         return rc;
1871 }
1872
1873 static void xillybus_cleanup_chrdev(struct xilly_endpoint *endpoint)
1874 {
1875         int minor;
1876
1877         for (minor = endpoint->lowest_minor;
1878              minor < (endpoint->lowest_minor + endpoint->num_channels);
1879              minor++)
1880                 device_destroy(xillybus_class, MKDEV(endpoint->major, minor));
1881         cdev_del(&endpoint->cdev);
1882         unregister_chrdev_region(MKDEV(endpoint->major,
1883                                        endpoint->lowest_minor),
1884                                  endpoint->num_channels);
1885
1886         dev_info(endpoint->dev, "Removed %d device files.\n",
1887                  endpoint->num_channels);
1888 }
1889
1890 struct xilly_endpoint *xillybus_init_endpoint(struct pci_dev *pdev,
1891                                               struct device *dev,
1892                                               struct xilly_endpoint_hardware
1893                                               *ephw)
1894 {
1895         struct xilly_endpoint *endpoint;
1896
1897         endpoint = devm_kzalloc(dev, sizeof(*endpoint), GFP_KERNEL);
1898         if (!endpoint)
1899                 return NULL;
1900
1901         endpoint->pdev = pdev;
1902         endpoint->dev = dev;
1903         endpoint->ephw = ephw;
1904         endpoint->msg_counter = 0x0b;
1905         endpoint->failed_messages = 0;
1906         endpoint->fatal_error = 0;
1907
1908         init_waitqueue_head(&endpoint->ep_wait);
1909         mutex_init(&endpoint->register_mutex);
1910
1911         return endpoint;
1912 }
1913 EXPORT_SYMBOL(xillybus_init_endpoint);
1914
1915 static int xilly_quiesce(struct xilly_endpoint *endpoint)
1916 {
1917         long t;
1918
1919         endpoint->idtlen = -1;
1920
1921         iowrite32((u32) (endpoint->dma_using_dac & 0x0001),
1922                   endpoint->registers + fpga_dma_control_reg);
1923
1924         t = wait_event_interruptible_timeout(endpoint->ep_wait,
1925                                              (endpoint->idtlen >= 0),
1926                                              XILLY_TIMEOUT);
1927         if (t <= 0) {
1928                 dev_err(endpoint->dev,
1929                         "Failed to quiesce the device on exit.\n");
1930                 return -ENODEV;
1931         }
1932         return 0;
1933 }
1934
1935 int xillybus_endpoint_discovery(struct xilly_endpoint *endpoint)
1936 {
1937         int rc;
1938         long t;
1939
1940         void *bootstrap_resources;
1941         int idtbuffersize = (1 << PAGE_SHIFT);
1942         struct device *dev = endpoint->dev;
1943
1944         /*
1945          * The bogus IDT is used during bootstrap for allocating the initial
1946          * message buffer, and then the message buffer and space for the IDT
1947          * itself. The initial message buffer is of a single page's size, but
1948          * it's soon replaced with a more modest one (and memory is freed).
1949          */
1950
1951         unsigned char bogus_idt[8] = { 1, 224, (PAGE_SHIFT)-2, 0,
1952                                        3, 192, PAGE_SHIFT, 0 };
1953         struct xilly_idt_handle idt_handle;
1954
1955         /*
1956          * Writing the value 0x00000001 to Endianness register signals which
1957          * endianness this processor is using, so the FPGA can swap words as
1958          * necessary.
1959          */
1960
1961         iowrite32(1, endpoint->registers + fpga_endian_reg);
1962
1963         /* Bootstrap phase I: Allocate temporary message buffer */
1964
1965         bootstrap_resources = devres_open_group(dev, NULL, GFP_KERNEL);
1966         if (!bootstrap_resources)
1967                 return -ENOMEM;
1968
1969         endpoint->num_channels = 0;
1970
1971         rc = xilly_setupchannels(endpoint, bogus_idt, 1);
1972         if (rc)
1973                 return rc;
1974
1975         /* Clear the message subsystem (and counter in particular) */
1976         iowrite32(0x04, endpoint->registers + fpga_msg_ctrl_reg);
1977
1978         endpoint->idtlen = -1;
1979
1980         /*
1981          * Set DMA 32/64 bit mode, quiesce the device (?!) and get IDT
1982          * buffer size.
1983          */
1984         iowrite32((u32) (endpoint->dma_using_dac & 0x0001),
1985                   endpoint->registers + fpga_dma_control_reg);
1986
1987         t = wait_event_interruptible_timeout(endpoint->ep_wait,
1988                                              (endpoint->idtlen >= 0),
1989                                              XILLY_TIMEOUT);
1990         if (t <= 0) {
1991                 dev_err(endpoint->dev, "No response from FPGA. Aborting.\n");
1992                 return -ENODEV;
1993         }
1994
1995         /* Enable DMA */
1996         iowrite32((u32) (0x0002 | (endpoint->dma_using_dac & 0x0001)),
1997                   endpoint->registers + fpga_dma_control_reg);
1998
1999         /* Bootstrap phase II: Allocate buffer for IDT and obtain it */
2000         while (endpoint->idtlen >= idtbuffersize) {
2001                 idtbuffersize *= 2;
2002                 bogus_idt[6]++;
2003         }
2004
2005         endpoint->num_channels = 1;
2006
2007         rc = xilly_setupchannels(endpoint, bogus_idt, 2);
2008         if (rc)
2009                 goto failed_idt;
2010
2011         rc = xilly_obtain_idt(endpoint);
2012         if (rc)
2013                 goto failed_idt;
2014
2015         rc = xilly_scan_idt(endpoint, &idt_handle);
2016         if (rc)
2017                 goto failed_idt;
2018
2019         devres_close_group(dev, bootstrap_resources);
2020
2021         /* Bootstrap phase III: Allocate buffers according to IDT */
2022
2023         rc = xilly_setupchannels(endpoint,
2024                                  idt_handle.chandesc,
2025                                  idt_handle.entries);
2026         if (rc)
2027                 goto failed_idt;
2028
2029         /*
2030          * endpoint is now completely configured. We put it on the list
2031          * available to open() before registering the char device(s)
2032          */
2033
2034         mutex_lock(&ep_list_lock);
2035         list_add_tail(&endpoint->ep_list, &list_of_endpoints);
2036         mutex_unlock(&ep_list_lock);
2037
2038         rc = xillybus_init_chrdev(endpoint, idt_handle.idt);
2039         if (rc)
2040                 goto failed_chrdevs;
2041
2042         devres_release_group(dev, bootstrap_resources);
2043
2044         return 0;
2045
2046 failed_chrdevs:
2047         mutex_lock(&ep_list_lock);
2048         list_del(&endpoint->ep_list);
2049         mutex_unlock(&ep_list_lock);
2050
2051 failed_idt:
2052         xilly_quiesce(endpoint);
2053         flush_workqueue(xillybus_wq);
2054
2055         return rc;
2056 }
2057 EXPORT_SYMBOL(xillybus_endpoint_discovery);
2058
2059 void xillybus_endpoint_remove(struct xilly_endpoint *endpoint)
2060 {
2061         xillybus_cleanup_chrdev(endpoint);
2062
2063         mutex_lock(&ep_list_lock);
2064         list_del(&endpoint->ep_list);
2065         mutex_unlock(&ep_list_lock);
2066
2067         xilly_quiesce(endpoint);
2068
2069         /*
2070          * Flushing is done upon endpoint release to prevent access to memory
2071          * just about to be released. This makes the quiesce complete.
2072          */
2073         flush_workqueue(xillybus_wq);
2074 }
2075 EXPORT_SYMBOL(xillybus_endpoint_remove);
2076
2077 static int __init xillybus_init(void)
2078 {
2079         mutex_init(&ep_list_lock);
2080
2081         xillybus_class = class_create(THIS_MODULE, xillyname);
2082         if (IS_ERR(xillybus_class))
2083                 return PTR_ERR(xillybus_class);
2084
2085         xillybus_wq = alloc_workqueue(xillyname, 0, 0);
2086         if (!xillybus_wq) {
2087                 class_destroy(xillybus_class);
2088                 return -ENOMEM;
2089         }
2090
2091         return 0;
2092 }
2093
2094 static void __exit xillybus_exit(void)
2095 {
2096         /* flush_workqueue() was called for each endpoint released */
2097         destroy_workqueue(xillybus_wq);
2098
2099         class_destroy(xillybus_class);
2100 }
2101
2102 module_init(xillybus_init);
2103 module_exit(xillybus_exit);