drivers/staging/kpc2000/kpc_dma/fileops.c

   1 /* SPDX-License-Identifier: GPL-2.0+ */
   2 #include <linux/module.h>
   3 #include <linux/init.h>
   4 #include <linux/mm.h>
   5 #include <linux/kernel.h>   /* printk() */
   6 #include <linux/slab.h>     /* kmalloc() */
   7 #include <linux/fs.h>       /* everything... */
   8 #include <linux/errno.h>    /* error codes */
   9 #include <linux/types.h>    /* size_t */
  10 #include <linux/cdev.h>
  11 #include <linux/uaccess.h>  /* copy_*_user */
  12 #include <linux/highmem.h>
  13 #include <linux/pagemap.h>
  14 #include "kpc_dma_driver.h"
  15 #include "uapi.h"
  16
  17 /**********  Helper Functions  **********/
  18 static inline
  19 unsigned int  count_pages(unsigned long iov_base, size_t iov_len)
  20 {
  21         unsigned long first = (iov_base             & PAGE_MASK) >> PAGE_SHIFT;
  22         unsigned long last  = ((iov_base+iov_len-1) & PAGE_MASK) >> PAGE_SHIFT;
  23
  24         return last - first + 1;
  25 }
  26
  27 static inline
  28 unsigned int  count_parts_for_sge(struct scatterlist *sg)
  29 {
  30         return DIV_ROUND_UP(sg_dma_len(sg), 0x80000);
  31 }
  32
  33 /**********  Transfer Helpers  **********/
  34 static int kpc_dma_transfer(struct dev_private_data *priv,
  35                             unsigned long iov_base, size_t iov_len)
  36 {
  37         unsigned int i = 0;
  38         long rv = 0;
  39         struct kpc_dma_device *ldev;
  40         struct aio_cb_data *acd;
  41         DECLARE_COMPLETION_ONSTACK(done);
  42         u32 desc_needed = 0;
  43         struct scatterlist *sg;
  44         u32 num_descrs_avail;
  45         struct kpc_dma_descriptor *desc;
  46         unsigned int pcnt;
  47         unsigned int p;
  48         u64 card_addr;
  49         u64 dma_addr;
  50         u64 user_ctl;
  51
  52         BUG_ON(priv == NULL);
  53         ldev = priv->ldev;
  54         BUG_ON(ldev == NULL);
  55
  56         acd = kzalloc(sizeof(*acd), GFP_KERNEL);
  57         if (!acd) {
  58                 dev_err(&priv->ldev->pldev->dev, "Couldn't kmalloc space for for the aio data\n");
  59                 return -ENOMEM;
  60         }
  61         memset(acd, 0x66, sizeof(struct aio_cb_data));
  62
  63         acd->priv = priv;
  64         acd->ldev = priv->ldev;
  65         acd->cpl = &done;
  66         acd->flags = 0;
  67         acd->len = iov_len;
  68         acd->page_count = count_pages(iov_base, iov_len);
  69
  70         // Allocate an array of page pointers
  71         acd->user_pages = kzalloc(sizeof(struct page *) * acd->page_count, GFP_KERNEL);
  72         if (!acd->user_pages) {
  73                 dev_err(&priv->ldev->pldev->dev, "Couldn't kmalloc space for for the page pointers\n");
  74                 rv = -ENOMEM;
  75                 goto err_alloc_userpages;
  76         }
  77
  78         // Lock the user buffer pages in memory, and hold on to the page pointers (for the sglist)
  79         down_read(&current->mm->mmap_sem);      /*  get memory map semaphore */
  80         rv = get_user_pages(iov_base, acd->page_count, FOLL_TOUCH | FOLL_WRITE | FOLL_GET, acd->user_pages, NULL);
  81         up_read(&current->mm->mmap_sem);        /*  release the semaphore */
  82         if (rv != acd->page_count) {
  83                 dev_err(&priv->ldev->pldev->dev, "Couldn't get_user_pages (%ld)\n", rv);
  84                 goto err_get_user_pages;
  85         }
  86
  87         // Allocate and setup the sg_table (scatterlist entries)
  88         rv = sg_alloc_table_from_pages(&acd->sgt, acd->user_pages, acd->page_count, iov_base & (PAGE_SIZE-1), iov_len, GFP_KERNEL);
  89         if (rv) {
  90                 dev_err(&priv->ldev->pldev->dev, "Couldn't alloc sg_table (%ld)\n", rv);
  91                 goto err_alloc_sg_table;
  92         }
  93
  94         // Setup the DMA mapping for all the sg entries
  95         acd->mapped_entry_count = dma_map_sg(&ldev->pldev->dev, acd->sgt.sgl, acd->sgt.nents, ldev->dir);
  96         if (acd->mapped_entry_count <= 0) {
  97                 dev_err(&priv->ldev->pldev->dev, "Couldn't dma_map_sg (%d)\n", acd->mapped_entry_count);
  98                 goto err_dma_map_sg;
  99         }
 100
 101         // Calculate how many descriptors are actually needed for this transfer.
 102         for_each_sg(acd->sgt.sgl, sg, acd->mapped_entry_count, i) {
 103                 desc_needed += count_parts_for_sge(sg);
 104         }
 105
 106         lock_engine(ldev);
 107
 108         // Figoure out how many descriptors are available and return an error if there aren't enough
 109         num_descrs_avail = count_descriptors_available(ldev);
 110         dev_dbg(&priv->ldev->pldev->dev, "    mapped_entry_count = %d    num_descrs_needed = %d    num_descrs_avail = %d\n", acd->mapped_entry_count, desc_needed, num_descrs_avail);
 111         if (desc_needed >= ldev->desc_pool_cnt) {
 112                 dev_warn(&priv->ldev->pldev->dev, "    mapped_entry_count = %d    num_descrs_needed = %d    num_descrs_avail = %d    TOO MANY to ever complete!\n", acd->mapped_entry_count, desc_needed, num_descrs_avail);
 113                 rv = -EAGAIN;
 114                 goto err_descr_too_many;
 115         }
 116         if (desc_needed > num_descrs_avail) {
 117                 dev_warn(&priv->ldev->pldev->dev, "    mapped_entry_count = %d    num_descrs_needed = %d    num_descrs_avail = %d    Too many to complete right now.\n", acd->mapped_entry_count, desc_needed, num_descrs_avail);
 118                 rv = -EMSGSIZE;
 119                 goto err_descr_too_many;
 120         }
 121
 122         // Loop through all the sg table entries and fill out a descriptor for each one.
 123         desc = ldev->desc_next;
 124         card_addr = acd->priv->card_addr;
 125         for_each_sg(acd->sgt.sgl, sg, acd->mapped_entry_count, i) {
 126                 pcnt = count_parts_for_sge(sg);
 127                 for (p = 0 ; p < pcnt ; p++) {
 128                         // Fill out the descriptor
 129                         BUG_ON(desc == NULL);
 130                         clear_desc(desc);
 131                         if (p != pcnt-1) {
 132                                 desc->DescByteCount = 0x80000;
 133                         } else {
 134                                 desc->DescByteCount = sg_dma_len(sg) - (p * 0x80000);
 135                         }
 136                         desc->DescBufferByteCount = desc->DescByteCount;
 137
 138                         desc->DescControlFlags |= DMA_DESC_CTL_IRQONERR;
 139                         if (i == 0 && p == 0)
 140                                 desc->DescControlFlags |= DMA_DESC_CTL_SOP;
 141                         if (i == acd->mapped_entry_count-1 && p == pcnt-1)
 142                                 desc->DescControlFlags |= DMA_DESC_CTL_EOP | DMA_DESC_CTL_IRQONDONE;
 143
 144                         desc->DescCardAddrLS = (card_addr & 0xFFFFFFFF);
 145                         desc->DescCardAddrMS = (card_addr >> 32) & 0xF;
 146                         card_addr += desc->DescByteCount;
 147
 148                         dma_addr  = sg_dma_address(sg) + (p * 0x80000);
 149                         desc->DescSystemAddrLS = (dma_addr & 0x00000000FFFFFFFF) >>  0;
 150                         desc->DescSystemAddrMS = (dma_addr & 0xFFFFFFFF00000000) >> 32;
 151
 152                         user_ctl = acd->priv->user_ctl;
 153                         if (i == acd->mapped_entry_count-1 && p == pcnt-1) {
 154                                 user_ctl = acd->priv->user_ctl_last;
 155                         }
 156                         desc->DescUserControlLS = (user_ctl & 0x00000000FFFFFFFF) >>  0;
 157                         desc->DescUserControlMS = (user_ctl & 0xFFFFFFFF00000000) >> 32;
 158
 159                         if (i == acd->mapped_entry_count-1 && p == pcnt-1)
 160                                 desc->acd = acd;
 161
 162                         dev_dbg(&priv->ldev->pldev->dev, "  Filled descriptor %p (acd = %p)\n", desc, desc->acd);
 163
 164                         ldev->desc_next = desc->Next;
 165                         desc = desc->Next;
 166                 }
 167         }
 168
 169         // Send the filled descriptors off to the hardware to process!
 170         SetEngineSWPtr(ldev, ldev->desc_next);
 171
 172         unlock_engine(ldev);
 173
 174         rv = wait_for_completion_interruptible(&done);
 175         /*
 176          * If the user aborted (rv == -ERESTARTSYS), we're no longer responsible
 177          * for cleaning up the acd
 178          */
 179         if (rv == -ERESTARTSYS)
 180                 acd->cpl = NULL;
 181         if (rv == 0) {
 182                 rv = acd->len;
 183                 kfree(acd);
 184         }
 185         return rv;
 186
 187  err_descr_too_many:
 188         unlock_engine(ldev);
 189         dma_unmap_sg(&ldev->pldev->dev, acd->sgt.sgl, acd->sgt.nents, ldev->dir);
 190         sg_free_table(&acd->sgt);
 191  err_dma_map_sg:
 192  err_alloc_sg_table:
 193         for (i = 0 ; i < acd->page_count ; i++) {
 194                 put_page(acd->user_pages[i]);
 195         }
 196  err_get_user_pages:
 197         kfree(acd->user_pages);
 198  err_alloc_userpages:
 199         kfree(acd);
 200         dev_dbg(&priv->ldev->pldev->dev, "%s returning with error %ld\n", __func__, rv);
 201         return rv;
 202 }
 203
 204 void  transfer_complete_cb(struct aio_cb_data *acd, size_t xfr_count, u32 flags)
 205 {
 206         unsigned int i;
 207
 208         BUG_ON(acd == NULL);
 209         BUG_ON(acd->user_pages == NULL);
 210         BUG_ON(acd->sgt.sgl == NULL);
 211         BUG_ON(acd->ldev == NULL);
 212         BUG_ON(acd->ldev->pldev == NULL);
 213
 214         for (i = 0 ; i < acd->page_count ; i++) {
 215                 if (!PageReserved(acd->user_pages[i])) {
 216                         set_page_dirty(acd->user_pages[i]);
 217                 }
 218         }
 219
 220         dma_unmap_sg(&acd->ldev->pldev->dev, acd->sgt.sgl, acd->sgt.nents, acd->ldev->dir);
 221
 222         for (i = 0 ; i < acd->page_count ; i++) {
 223                 put_page(acd->user_pages[i]);
 224         }
 225
 226         sg_free_table(&acd->sgt);
 227
 228         kfree(acd->user_pages);
 229
 230         acd->flags = flags;
 231
 232         if (acd->cpl) {
 233                 complete(acd->cpl);
 234         } else {
 235                 /*
 236                  * There's no completion, so we're responsible for cleaning up
 237                  * the acd
 238                  */
 239                 kfree(acd);
 240         }
 241 }
 242
 243 /**********  Fileops  **********/
 244 static
 245 int  kpc_dma_open(struct inode *inode, struct file *filp)
 246 {
 247         struct dev_private_data *priv;
 248         struct kpc_dma_device *ldev = kpc_dma_lookup_device(iminor(inode));
 249
 250         if (!ldev)
 251                 return -ENODEV;
 252
 253         if (!atomic_dec_and_test(&ldev->open_count)) {
 254                 atomic_inc(&ldev->open_count);
 255                 return -EBUSY; /* already open */
 256         }
 257
 258         priv = kzalloc(sizeof(struct dev_private_data), GFP_KERNEL);
 259         if (!priv)
 260                 return -ENOMEM;
 261
 262         priv->ldev = ldev;
 263         filp->private_data = priv;
 264
 265         return 0;
 266 }
 267
 268 static
 269 int  kpc_dma_close(struct inode *inode, struct file *filp)
 270 {
 271         struct kpc_dma_descriptor *cur;
 272         struct dev_private_data *priv = (struct dev_private_data *)filp->private_data;
 273         struct kpc_dma_device *eng = priv->ldev;
 274
 275         lock_engine(eng);
 276
 277         stop_dma_engine(eng);
 278
 279         cur = eng->desc_completed->Next;
 280         while (cur != eng->desc_next) {
 281                 dev_dbg(&eng->pldev->dev, "Aborting descriptor %p (acd = %p)\n", cur, cur->acd);
 282                 if (cur->DescControlFlags & DMA_DESC_CTL_EOP) {
 283                         if (cur->acd)
 284                                 transfer_complete_cb(cur->acd, 0, ACD_FLAG_ABORT);
 285                 }
 286
 287                 clear_desc(cur);
 288                 eng->desc_completed = cur;
 289
 290                 cur = cur->Next;
 291         }
 292
 293         start_dma_engine(eng);
 294
 295         unlock_engine(eng);
 296
 297         atomic_inc(&priv->ldev->open_count); /* release the device */
 298         kfree(priv);
 299         return 0;
 300 }
 301
 302 static
 303 ssize_t  kpc_dma_read(struct file *filp,       char __user *user_buf, size_t count, loff_t *ppos)
 304 {
 305         struct dev_private_data *priv = (struct dev_private_data *)filp->private_data;
 306
 307         if (priv->ldev->dir != DMA_FROM_DEVICE)
 308                 return -EMEDIUMTYPE;
 309
 310         return kpc_dma_transfer(priv, (unsigned long)user_buf, count);
 311 }
 312
 313 static
 314 ssize_t  kpc_dma_write(struct file *filp, const char __user *user_buf, size_t count, loff_t *ppos)
 315 {
 316         struct dev_private_data *priv = (struct dev_private_data *)filp->private_data;
 317
 318         if (priv->ldev->dir != DMA_TO_DEVICE)
 319                 return -EMEDIUMTYPE;
 320
 321         return kpc_dma_transfer(priv, (unsigned long)user_buf, count);
 322 }
 323
 324 static
 325 long  kpc_dma_ioctl(struct file *filp, unsigned int ioctl_num, unsigned long ioctl_param)
 326 {
 327         struct dev_private_data *priv = (struct dev_private_data *)filp->private_data;
 328
 329         switch (ioctl_num) {
 330         case KND_IOCTL_SET_CARD_ADDR:
 331                 priv->card_addr  = ioctl_param; return priv->card_addr;
 332         case KND_IOCTL_SET_USER_CTL:
 333                 priv->user_ctl   = ioctl_param; return priv->user_ctl;
 334         case KND_IOCTL_SET_USER_CTL_LAST:
 335                 priv->user_ctl_last = ioctl_param; return priv->user_ctl_last;
 336         case KND_IOCTL_GET_USER_STS:
 337                 return priv->user_sts;
 338         }
 339
 340         return -ENOTTY;
 341 }
 342
 343 const struct file_operations  kpc_dma_fops = {
 344         .owner      = THIS_MODULE,
 345         .open           = kpc_dma_open,
 346         .release        = kpc_dma_close,
 347         .read           = kpc_dma_read,
 348         .write          = kpc_dma_write,
 349         .unlocked_ioctl = kpc_dma_ioctl,
 350 };
 351