drivers/crypto/nx/nx-842-pseries.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * Driver for IBM Power 842 compression accelerator
   4  *
   5  * Copyright (C) IBM Corporation, 2012
   6  *
   7  * Authors: Robert Jennings <rcj@linux.vnet.ibm.com>
   8  *          Seth Jennings <sjenning@linux.vnet.ibm.com>
   9  */
  10
  11 #include <asm/vio.h>
  12
  13 #include "nx-842.h"
  14 #include "nx_csbcpb.h" /* struct nx_csbcpb */
  15
  16 MODULE_LICENSE("GPL");
  17 MODULE_AUTHOR("Robert Jennings <rcj@linux.vnet.ibm.com>");
  18 MODULE_DESCRIPTION("842 H/W Compression driver for IBM Power processors");
  19 MODULE_ALIAS_CRYPTO("842");
  20 MODULE_ALIAS_CRYPTO("842-nx");
  21
  22 static struct nx842_constraints nx842_pseries_constraints = {
  23         .alignment =    DDE_BUFFER_ALIGN,
  24         .multiple =     DDE_BUFFER_LAST_MULT,
  25         .minimum =      DDE_BUFFER_LAST_MULT,
  26         .maximum =      PAGE_SIZE, /* dynamic, max_sync_size */
  27 };
  28
  29 static int check_constraints(unsigned long buf, unsigned int *len, bool in)
  30 {
  31         if (!IS_ALIGNED(buf, nx842_pseries_constraints.alignment)) {
  32                 pr_debug("%s buffer 0x%lx not aligned to 0x%x\n",
  33                          in ? "input" : "output", buf,
  34                          nx842_pseries_constraints.alignment);
  35                 return -EINVAL;
  36         }
  37         if (*len % nx842_pseries_constraints.multiple) {
  38                 pr_debug("%s buffer len 0x%x not multiple of 0x%x\n",
  39                          in ? "input" : "output", *len,
  40                          nx842_pseries_constraints.multiple);
  41                 if (in)
  42                         return -EINVAL;
  43                 *len = round_down(*len, nx842_pseries_constraints.multiple);
  44         }
  45         if (*len < nx842_pseries_constraints.minimum) {
  46                 pr_debug("%s buffer len 0x%x under minimum 0x%x\n",
  47                          in ? "input" : "output", *len,
  48                          nx842_pseries_constraints.minimum);
  49                 return -EINVAL;
  50         }
  51         if (*len > nx842_pseries_constraints.maximum) {
  52                 pr_debug("%s buffer len 0x%x over maximum 0x%x\n",
  53                          in ? "input" : "output", *len,
  54                          nx842_pseries_constraints.maximum);
  55                 if (in)
  56                         return -EINVAL;
  57                 *len = nx842_pseries_constraints.maximum;
  58         }
  59         return 0;
  60 }
  61
  62 /* I assume we need to align the CSB? */
  63 #define WORKMEM_ALIGN   (256)
  64
  65 struct nx842_workmem {
  66         /* scatterlist */
  67         char slin[4096];
  68         char slout[4096];
  69         /* coprocessor status/parameter block */
  70         struct nx_csbcpb csbcpb;
  71
  72         char padding[WORKMEM_ALIGN];
  73 } __aligned(WORKMEM_ALIGN);
  74
  75 /* Macros for fields within nx_csbcpb */
  76 /* Check the valid bit within the csbcpb valid field */
  77 #define NX842_CSBCBP_VALID_CHK(x) (x & BIT_MASK(7))
  78
  79 /* CE macros operate on the completion_extension field bits in the csbcpb.
  80  * CE0 0=full completion, 1=partial completion
  81  * CE1 0=CE0 indicates completion, 1=termination (output may be modified)
  82  * CE2 0=processed_bytes is source bytes, 1=processed_bytes is target bytes */
  83 #define NX842_CSBCPB_CE0(x)     (x & BIT_MASK(7))
  84 #define NX842_CSBCPB_CE1(x)     (x & BIT_MASK(6))
  85 #define NX842_CSBCPB_CE2(x)     (x & BIT_MASK(5))
  86
  87 /* The NX unit accepts data only on 4K page boundaries */
  88 #define NX842_HW_PAGE_SIZE      (4096)
  89 #define NX842_HW_PAGE_MASK      (~(NX842_HW_PAGE_SIZE-1))
  90
  91 struct ibm_nx842_counters {
  92         atomic64_t comp_complete;
  93         atomic64_t comp_failed;
  94         atomic64_t decomp_complete;
  95         atomic64_t decomp_failed;
  96         atomic64_t swdecomp;
  97         atomic64_t comp_times[32];
  98         atomic64_t decomp_times[32];
  99 };
 100
 101 static struct nx842_devdata {
 102         struct vio_dev *vdev;
 103         struct device *dev;
 104         struct ibm_nx842_counters *counters;
 105         unsigned int max_sg_len;
 106         unsigned int max_sync_size;
 107         unsigned int max_sync_sg;
 108 } __rcu *devdata;
 109 static DEFINE_SPINLOCK(devdata_mutex);
 110
 111 #define NX842_COUNTER_INC(_x) \
 112 static inline void nx842_inc_##_x( \
 113         const struct nx842_devdata *dev) { \
 114         if (dev) \
 115                 atomic64_inc(&dev->counters->_x); \
 116 }
 117 NX842_COUNTER_INC(comp_complete);
 118 NX842_COUNTER_INC(comp_failed);
 119 NX842_COUNTER_INC(decomp_complete);
 120 NX842_COUNTER_INC(decomp_failed);
 121 NX842_COUNTER_INC(swdecomp);
 122
 123 #define NX842_HIST_SLOTS 16
 124
 125 static void ibm_nx842_incr_hist(atomic64_t *times, unsigned int time)
 126 {
 127         int bucket = fls(time);
 128
 129         if (bucket)
 130                 bucket = min((NX842_HIST_SLOTS - 1), bucket - 1);
 131
 132         atomic64_inc(&times[bucket]);
 133 }
 134
 135 /* NX unit operation flags */
 136 #define NX842_OP_COMPRESS       0x0
 137 #define NX842_OP_CRC            0x1
 138 #define NX842_OP_DECOMPRESS     0x2
 139 #define NX842_OP_COMPRESS_CRC   (NX842_OP_COMPRESS | NX842_OP_CRC)
 140 #define NX842_OP_DECOMPRESS_CRC (NX842_OP_DECOMPRESS | NX842_OP_CRC)
 141 #define NX842_OP_ASYNC          (1<<23)
 142 #define NX842_OP_NOTIFY         (1<<22)
 143 #define NX842_OP_NOTIFY_INT(x)  ((x & 0xff)<<8)
 144
 145 static unsigned long nx842_get_desired_dma(struct vio_dev *viodev)
 146 {
 147         /* No use of DMA mappings within the driver. */
 148         return 0;
 149 }
 150
 151 struct nx842_slentry {
 152         __be64 ptr; /* Real address (use __pa()) */
 153         __be64 len;
 154 };
 155
 156 /* pHyp scatterlist entry */
 157 struct nx842_scatterlist {
 158         int entry_nr; /* number of slentries */
 159         struct nx842_slentry *entries; /* ptr to array of slentries */
 160 };
 161
 162 /* Does not include sizeof(entry_nr) in the size */
 163 static inline unsigned long nx842_get_scatterlist_size(
 164                                 struct nx842_scatterlist *sl)
 165 {
 166         return sl->entry_nr * sizeof(struct nx842_slentry);
 167 }
 168
 169 static int nx842_build_scatterlist(unsigned long buf, int len,
 170                         struct nx842_scatterlist *sl)
 171 {
 172         unsigned long entrylen;
 173         struct nx842_slentry *entry;
 174
 175         sl->entry_nr = 0;
 176
 177         entry = sl->entries;
 178         while (len) {
 179                 entry->ptr = cpu_to_be64(nx842_get_pa((void *)buf));
 180                 entrylen = min_t(int, len,
 181                                  LEN_ON_SIZE(buf, NX842_HW_PAGE_SIZE));
 182                 entry->len = cpu_to_be64(entrylen);
 183
 184                 len -= entrylen;
 185                 buf += entrylen;
 186
 187                 sl->entry_nr++;
 188                 entry++;
 189         }
 190
 191         return 0;
 192 }
 193
 194 static int nx842_validate_result(struct device *dev,
 195         struct cop_status_block *csb)
 196 {
 197         /* The csb must be valid after returning from vio_h_cop_sync */
 198         if (!NX842_CSBCBP_VALID_CHK(csb->valid)) {
 199                 dev_err(dev, "%s: cspcbp not valid upon completion.\n",
 200                                 __func__);
 201                 dev_dbg(dev, "valid:0x%02x cs:0x%02x cc:0x%02x ce:0x%02x\n",
 202                                 csb->valid,
 203                                 csb->crb_seq_number,
 204                                 csb->completion_code,
 205                                 csb->completion_extension);
 206                 dev_dbg(dev, "processed_bytes:%d address:0x%016lx\n",
 207                                 be32_to_cpu(csb->processed_byte_count),
 208                                 (unsigned long)be64_to_cpu(csb->address));
 209                 return -EIO;
 210         }
 211
 212         /* Check return values from the hardware in the CSB */
 213         switch (csb->completion_code) {
 214         case 0: /* Completed without error */
 215                 break;
 216         case 64: /* Compression ok, but output larger than input */
 217                 dev_dbg(dev, "%s: output size larger than input size\n",
 218                                         __func__);
 219                 break;
 220         case 13: /* Output buffer too small */
 221                 dev_dbg(dev, "%s: Out of space in output buffer\n",
 222                                         __func__);
 223                 return -ENOSPC;
 224         case 65: /* Calculated CRC doesn't match the passed value */
 225                 dev_dbg(dev, "%s: CRC mismatch for decompression\n",
 226                                         __func__);
 227                 return -EINVAL;
 228         case 66: /* Input data contains an illegal template field */
 229         case 67: /* Template indicates data past the end of the input stream */
 230                 dev_dbg(dev, "%s: Bad data for decompression (code:%d)\n",
 231                                         __func__, csb->completion_code);
 232                 return -EINVAL;
 233         default:
 234                 dev_dbg(dev, "%s: Unspecified error (code:%d)\n",
 235                                         __func__, csb->completion_code);
 236                 return -EIO;
 237         }
 238
 239         /* Hardware sanity check */
 240         if (!NX842_CSBCPB_CE2(csb->completion_extension)) {
 241                 dev_err(dev, "%s: No error returned by hardware, but "
 242                                 "data returned is unusable, contact support.\n"
 243                                 "(Additional info: csbcbp->processed bytes "
 244                                 "does not specify processed bytes for the "
 245                                 "target buffer.)\n", __func__);
 246                 return -EIO;
 247         }
 248
 249         return 0;
 250 }
 251
 252 /**
 253  * nx842_pseries_compress - Compress data using the 842 algorithm
 254  *
 255  * Compression provide by the NX842 coprocessor on IBM Power systems.
 256  * The input buffer is compressed and the result is stored in the
 257  * provided output buffer.
 258  *
 259  * Upon return from this function @outlen contains the length of the
 260  * compressed data.  If there is an error then @outlen will be 0 and an
 261  * error will be specified by the return code from this function.
 262  *
 263  * @in: Pointer to input buffer
 264  * @inlen: Length of input buffer
 265  * @out: Pointer to output buffer
 266  * @outlen: Length of output buffer
 267  * @wrkmem: ptr to buffer for working memory, size determined by
 268  *          nx842_pseries_driver.workmem_size
 269  *
 270  * Returns:
 271  *   0          Success, output of length @outlen stored in the buffer at @out
 272  *   -ENOMEM    Unable to allocate internal buffers
 273  *   -ENOSPC    Output buffer is to small
 274  *   -EIO       Internal error
 275  *   -ENODEV    Hardware unavailable
 276  */
 277 static int nx842_pseries_compress(const unsigned char *in, unsigned int inlen,
 278                                   unsigned char *out, unsigned int *outlen,
 279                                   void *wmem)
 280 {
 281         struct nx842_devdata *local_devdata;
 282         struct device *dev = NULL;
 283         struct nx842_workmem *workmem;
 284         struct nx842_scatterlist slin, slout;
 285         struct nx_csbcpb *csbcpb;
 286         int ret = 0;
 287         unsigned long inbuf, outbuf;
 288         struct vio_pfo_op op = {
 289                 .done = NULL,
 290                 .handle = 0,
 291                 .timeout = 0,
 292         };
 293         unsigned long start = get_tb();
 294
 295         inbuf = (unsigned long)in;
 296         if (check_constraints(inbuf, &inlen, true))
 297                 return -EINVAL;
 298
 299         outbuf = (unsigned long)out;
 300         if (check_constraints(outbuf, outlen, false))
 301                 return -EINVAL;
 302
 303         rcu_read_lock();
 304         local_devdata = rcu_dereference(devdata);
 305         if (!local_devdata || !local_devdata->dev) {
 306                 rcu_read_unlock();
 307                 return -ENODEV;
 308         }
 309         dev = local_devdata->dev;
 310
 311         /* Init scatterlist */
 312         workmem = PTR_ALIGN(wmem, WORKMEM_ALIGN);
 313         slin.entries = (struct nx842_slentry *)workmem->slin;
 314         slout.entries = (struct nx842_slentry *)workmem->slout;
 315
 316         /* Init operation */
 317         op.flags = NX842_OP_COMPRESS_CRC;
 318         csbcpb = &workmem->csbcpb;
 319         memset(csbcpb, 0, sizeof(*csbcpb));
 320         op.csbcpb = nx842_get_pa(csbcpb);
 321
 322         if ((inbuf & NX842_HW_PAGE_MASK) ==
 323             ((inbuf + inlen - 1) & NX842_HW_PAGE_MASK)) {
 324                 /* Create direct DDE */
 325                 op.in = nx842_get_pa((void *)inbuf);
 326                 op.inlen = inlen;
 327         } else {
 328                 /* Create indirect DDE (scatterlist) */
 329                 nx842_build_scatterlist(inbuf, inlen, &slin);
 330                 op.in = nx842_get_pa(slin.entries);
 331                 op.inlen = -nx842_get_scatterlist_size(&slin);
 332         }
 333
 334         if ((outbuf & NX842_HW_PAGE_MASK) ==
 335             ((outbuf + *outlen - 1) & NX842_HW_PAGE_MASK)) {
 336                 /* Create direct DDE */
 337                 op.out = nx842_get_pa((void *)outbuf);
 338                 op.outlen = *outlen;
 339         } else {
 340                 /* Create indirect DDE (scatterlist) */
 341                 nx842_build_scatterlist(outbuf, *outlen, &slout);
 342                 op.out = nx842_get_pa(slout.entries);
 343                 op.outlen = -nx842_get_scatterlist_size(&slout);
 344         }
 345
 346         dev_dbg(dev, "%s: op.in %lx op.inlen %ld op.out %lx op.outlen %ld\n",
 347                 __func__, (unsigned long)op.in, (long)op.inlen,
 348                 (unsigned long)op.out, (long)op.outlen);
 349
 350         /* Send request to pHyp */
 351         ret = vio_h_cop_sync(local_devdata->vdev, &op);
 352
 353         /* Check for pHyp error */
 354         if (ret) {
 355                 dev_dbg(dev, "%s: vio_h_cop_sync error (ret=%d, hret=%ld)\n",
 356                         __func__, ret, op.hcall_err);
 357                 ret = -EIO;
 358                 goto unlock;
 359         }
 360
 361         /* Check for hardware error */
 362         ret = nx842_validate_result(dev, &csbcpb->csb);
 363         if (ret)
 364                 goto unlock;
 365
 366         *outlen = be32_to_cpu(csbcpb->csb.processed_byte_count);
 367         dev_dbg(dev, "%s: processed_bytes=%d\n", __func__, *outlen);
 368
 369 unlock:
 370         if (ret)
 371                 nx842_inc_comp_failed(local_devdata);
 372         else {
 373                 nx842_inc_comp_complete(local_devdata);
 374                 ibm_nx842_incr_hist(local_devdata->counters->comp_times,
 375                         (get_tb() - start) / tb_ticks_per_usec);
 376         }
 377         rcu_read_unlock();
 378         return ret;
 379 }
 380
 381 /**
 382  * nx842_pseries_decompress - Decompress data using the 842 algorithm
 383  *
 384  * Decompression provide by the NX842 coprocessor on IBM Power systems.
 385  * The input buffer is decompressed and the result is stored in the
 386  * provided output buffer.  The size allocated to the output buffer is
 387  * provided by the caller of this function in @outlen.  Upon return from
 388  * this function @outlen contains the length of the decompressed data.
 389  * If there is an error then @outlen will be 0 and an error will be
 390  * specified by the return code from this function.
 391  *
 392  * @in: Pointer to input buffer
 393  * @inlen: Length of input buffer
 394  * @out: Pointer to output buffer
 395  * @outlen: Length of output buffer
 396  * @wrkmem: ptr to buffer for working memory, size determined by
 397  *          nx842_pseries_driver.workmem_size
 398  *
 399  * Returns:
 400  *   0          Success, output of length @outlen stored in the buffer at @out
 401  *   -ENODEV    Hardware decompression device is unavailable
 402  *   -ENOMEM    Unable to allocate internal buffers
 403  *   -ENOSPC    Output buffer is to small
 404  *   -EINVAL    Bad input data encountered when attempting decompress
 405  *   -EIO       Internal error
 406  */
 407 static int nx842_pseries_decompress(const unsigned char *in, unsigned int inlen,
 408                                     unsigned char *out, unsigned int *outlen,
 409                                     void *wmem)
 410 {
 411         struct nx842_devdata *local_devdata;
 412         struct device *dev = NULL;
 413         struct nx842_workmem *workmem;
 414         struct nx842_scatterlist slin, slout;
 415         struct nx_csbcpb *csbcpb;
 416         int ret = 0;
 417         unsigned long inbuf, outbuf;
 418         struct vio_pfo_op op = {
 419                 .done = NULL,
 420                 .handle = 0,
 421                 .timeout = 0,
 422         };
 423         unsigned long start = get_tb();
 424
 425         /* Ensure page alignment and size */
 426         inbuf = (unsigned long)in;
 427         if (check_constraints(inbuf, &inlen, true))
 428                 return -EINVAL;
 429
 430         outbuf = (unsigned long)out;
 431         if (check_constraints(outbuf, outlen, false))
 432                 return -EINVAL;
 433
 434         rcu_read_lock();
 435         local_devdata = rcu_dereference(devdata);
 436         if (!local_devdata || !local_devdata->dev) {
 437                 rcu_read_unlock();
 438                 return -ENODEV;
 439         }
 440         dev = local_devdata->dev;
 441
 442         workmem = PTR_ALIGN(wmem, WORKMEM_ALIGN);
 443
 444         /* Init scatterlist */
 445         slin.entries = (struct nx842_slentry *)workmem->slin;
 446         slout.entries = (struct nx842_slentry *)workmem->slout;
 447
 448         /* Init operation */
 449         op.flags = NX842_OP_DECOMPRESS_CRC;
 450         csbcpb = &workmem->csbcpb;
 451         memset(csbcpb, 0, sizeof(*csbcpb));
 452         op.csbcpb = nx842_get_pa(csbcpb);
 453
 454         if ((inbuf & NX842_HW_PAGE_MASK) ==
 455             ((inbuf + inlen - 1) & NX842_HW_PAGE_MASK)) {
 456                 /* Create direct DDE */
 457                 op.in = nx842_get_pa((void *)inbuf);
 458                 op.inlen = inlen;
 459         } else {
 460                 /* Create indirect DDE (scatterlist) */
 461                 nx842_build_scatterlist(inbuf, inlen, &slin);
 462                 op.in = nx842_get_pa(slin.entries);
 463                 op.inlen = -nx842_get_scatterlist_size(&slin);
 464         }
 465
 466         if ((outbuf & NX842_HW_PAGE_MASK) ==
 467             ((outbuf + *outlen - 1) & NX842_HW_PAGE_MASK)) {
 468                 /* Create direct DDE */
 469                 op.out = nx842_get_pa((void *)outbuf);
 470                 op.outlen = *outlen;
 471         } else {
 472                 /* Create indirect DDE (scatterlist) */
 473                 nx842_build_scatterlist(outbuf, *outlen, &slout);
 474                 op.out = nx842_get_pa(slout.entries);
 475                 op.outlen = -nx842_get_scatterlist_size(&slout);
 476         }
 477
 478         dev_dbg(dev, "%s: op.in %lx op.inlen %ld op.out %lx op.outlen %ld\n",
 479                 __func__, (unsigned long)op.in, (long)op.inlen,
 480                 (unsigned long)op.out, (long)op.outlen);
 481
 482         /* Send request to pHyp */
 483         ret = vio_h_cop_sync(local_devdata->vdev, &op);
 484
 485         /* Check for pHyp error */
 486         if (ret) {
 487                 dev_dbg(dev, "%s: vio_h_cop_sync error (ret=%d, hret=%ld)\n",
 488                         __func__, ret, op.hcall_err);
 489                 goto unlock;
 490         }
 491
 492         /* Check for hardware error */
 493         ret = nx842_validate_result(dev, &csbcpb->csb);
 494         if (ret)
 495                 goto unlock;
 496
 497         *outlen = be32_to_cpu(csbcpb->csb.processed_byte_count);
 498
 499 unlock:
 500         if (ret)
 501                 /* decompress fail */
 502                 nx842_inc_decomp_failed(local_devdata);
 503         else {
 504                 nx842_inc_decomp_complete(local_devdata);
 505                 ibm_nx842_incr_hist(local_devdata->counters->decomp_times,
 506                         (get_tb() - start) / tb_ticks_per_usec);
 507         }
 508
 509         rcu_read_unlock();
 510         return ret;
 511 }
 512
 513 /**
 514  * nx842_OF_set_defaults -- Set default (disabled) values for devdata
 515  *
 516  * @devdata - struct nx842_devdata to update
 517  *
 518  * Returns:
 519  *  0 on success
 520  *  -ENOENT if @devdata ptr is NULL
 521  */
 522 static int nx842_OF_set_defaults(struct nx842_devdata *devdata)
 523 {
 524         if (devdata) {
 525                 devdata->max_sync_size = 0;
 526                 devdata->max_sync_sg = 0;
 527                 devdata->max_sg_len = 0;
 528                 return 0;
 529         } else
 530                 return -ENOENT;
 531 }
 532
 533 /**
 534  * nx842_OF_upd_status -- Check the device info from OF status prop
 535  *
 536  * The status property indicates if the accelerator is enabled.  If the
 537  * device is in the OF tree it indicates that the hardware is present.
 538  * The status field indicates if the device is enabled when the status
 539  * is 'okay'.  Otherwise the device driver will be disabled.
 540  *
 541  * @prop - struct property point containing the maxsyncop for the update
 542  *
 543  * Returns:
 544  *  0 - Device is available
 545  *  -ENODEV - Device is not available
 546  */
 547 static int nx842_OF_upd_status(struct property *prop)
 548 {
 549         const char *status = (const char *)prop->value;
 550
 551         if (!strncmp(status, "okay", (size_t)prop->length))
 552                 return 0;
 553         if (!strncmp(status, "disabled", (size_t)prop->length))
 554                 return -ENODEV;
 555         dev_info(devdata->dev, "%s: unknown status '%s'\n", __func__, status);
 556
 557         return -EINVAL;
 558 }
 559
 560 /**
 561  * nx842_OF_upd_maxsglen -- Update the device info from OF maxsglen prop
 562  *
 563  * Definition of the 'ibm,max-sg-len' OF property:
 564  *  This field indicates the maximum byte length of a scatter list
 565  *  for the platform facility. It is a single cell encoded as with encode-int.
 566  *
 567  * Example:
 568  *  # od -x ibm,max-sg-len
 569  *  0000000 0000 0ff0
 570  *
 571  *  In this example, the maximum byte length of a scatter list is
 572  *  0x0ff0 (4,080).
 573  *
 574  * @devdata - struct nx842_devdata to update
 575  * @prop - struct property point containing the maxsyncop for the update
 576  *
 577  * Returns:
 578  *  0 on success
 579  *  -EINVAL on failure
 580  */
 581 static int nx842_OF_upd_maxsglen(struct nx842_devdata *devdata,
 582                                         struct property *prop) {
 583         int ret = 0;
 584         const unsigned int maxsglen = of_read_number(prop->value, 1);
 585
 586         if (prop->length != sizeof(maxsglen)) {
 587                 dev_err(devdata->dev, "%s: unexpected format for ibm,max-sg-len property\n", __func__);
 588                 dev_dbg(devdata->dev, "%s: ibm,max-sg-len is %d bytes long, expected %lu bytes\n", __func__,
 589                                 prop->length, sizeof(maxsglen));
 590                 ret = -EINVAL;
 591         } else {
 592                 devdata->max_sg_len = min_t(unsigned int,
 593                                             maxsglen, NX842_HW_PAGE_SIZE);
 594         }
 595
 596         return ret;
 597 }
 598
 599 /**
 600  * nx842_OF_upd_maxsyncop -- Update the device info from OF maxsyncop prop
 601  *
 602  * Definition of the 'ibm,max-sync-cop' OF property:
 603  *  Two series of cells.  The first series of cells represents the maximums
 604  *  that can be synchronously compressed. The second series of cells
 605  *  represents the maximums that can be synchronously decompressed.
 606  *  1. The first cell in each series contains the count of the number of
 607  *     data length, scatter list elements pairs that follow – each being
 608  *     of the form
 609  *    a. One cell data byte length
 610  *    b. One cell total number of scatter list elements
 611  *
 612  * Example:
 613  *  # od -x ibm,max-sync-cop
 614  *  0000000 0000 0001 0000 1000 0000 01fe 0000 0001
 615  *  0000020 0000 1000 0000 01fe
 616  *
 617  *  In this example, compression supports 0x1000 (4,096) data byte length
 618  *  and 0x1fe (510) total scatter list elements.  Decompression supports
 619  *  0x1000 (4,096) data byte length and 0x1f3 (510) total scatter list
 620  *  elements.
 621  *
 622  * @devdata - struct nx842_devdata to update
 623  * @prop - struct property point containing the maxsyncop for the update
 624  *
 625  * Returns:
 626  *  0 on success
 627  *  -EINVAL on failure
 628  */
 629 static int nx842_OF_upd_maxsyncop(struct nx842_devdata *devdata,
 630                                         struct property *prop) {
 631         int ret = 0;
 632         unsigned int comp_data_limit, decomp_data_limit;
 633         unsigned int comp_sg_limit, decomp_sg_limit;
 634         const struct maxsynccop_t {
 635                 __be32 comp_elements;
 636                 __be32 comp_data_limit;
 637                 __be32 comp_sg_limit;
 638                 __be32 decomp_elements;
 639                 __be32 decomp_data_limit;
 640                 __be32 decomp_sg_limit;
 641         } *maxsynccop;
 642
 643         if (prop->length != sizeof(*maxsynccop)) {
 644                 dev_err(devdata->dev, "%s: unexpected format for ibm,max-sync-cop property\n", __func__);
 645                 dev_dbg(devdata->dev, "%s: ibm,max-sync-cop is %d bytes long, expected %lu bytes\n", __func__, prop->length,
 646                                 sizeof(*maxsynccop));
 647                 ret = -EINVAL;
 648                 goto out;
 649         }
 650
 651         maxsynccop = (const struct maxsynccop_t *)prop->value;
 652         comp_data_limit = be32_to_cpu(maxsynccop->comp_data_limit);
 653         comp_sg_limit = be32_to_cpu(maxsynccop->comp_sg_limit);
 654         decomp_data_limit = be32_to_cpu(maxsynccop->decomp_data_limit);
 655         decomp_sg_limit = be32_to_cpu(maxsynccop->decomp_sg_limit);
 656
 657         /* Use one limit rather than separate limits for compression and
 658          * decompression. Set a maximum for this so as not to exceed the
 659          * size that the header can support and round the value down to
 660          * the hardware page size (4K) */
 661         devdata->max_sync_size = min(comp_data_limit, decomp_data_limit);
 662
 663         devdata->max_sync_size = min_t(unsigned int, devdata->max_sync_size,
 664                                         65536);
 665
 666         if (devdata->max_sync_size < 4096) {
 667                 dev_err(devdata->dev, "%s: hardware max data size (%u) is "
 668                                 "less than the driver minimum, unable to use "
 669                                 "the hardware device\n",
 670                                 __func__, devdata->max_sync_size);
 671                 ret = -EINVAL;
 672                 goto out;
 673         }
 674
 675         nx842_pseries_constraints.maximum = devdata->max_sync_size;
 676
 677         devdata->max_sync_sg = min(comp_sg_limit, decomp_sg_limit);
 678         if (devdata->max_sync_sg < 1) {
 679                 dev_err(devdata->dev, "%s: hardware max sg size (%u) is "
 680                                 "less than the driver minimum, unable to use "
 681                                 "the hardware device\n",
 682                                 __func__, devdata->max_sync_sg);
 683                 ret = -EINVAL;
 684                 goto out;
 685         }
 686
 687 out:
 688         return ret;
 689 }
 690
 691 /**
 692  *
 693  * nx842_OF_upd -- Handle OF properties updates for the device.
 694  *
 695  * Set all properties from the OF tree.  Optionally, a new property
 696  * can be provided by the @new_prop pointer to overwrite an existing value.
 697  * The device will remain disabled until all values are valid, this function
 698  * will return an error for updates unless all values are valid.
 699  *
 700  * @new_prop: If not NULL, this property is being updated.  If NULL, update
 701  *  all properties from the current values in the OF tree.
 702  *
 703  * Returns:
 704  *  0 - Success
 705  *  -ENOMEM - Could not allocate memory for new devdata structure
 706  *  -EINVAL - property value not found, new_prop is not a recognized
 707  *      property for the device or property value is not valid.
 708  *  -ENODEV - Device is not available
 709  */
 710 static int nx842_OF_upd(struct property *new_prop)
 711 {
 712         struct nx842_devdata *old_devdata = NULL;
 713         struct nx842_devdata *new_devdata = NULL;
 714         struct device_node *of_node = NULL;
 715         struct property *status = NULL;
 716         struct property *maxsglen = NULL;
 717         struct property *maxsyncop = NULL;
 718         int ret = 0;
 719         unsigned long flags;
 720
 721         new_devdata = kzalloc(sizeof(*new_devdata), GFP_NOFS);
 722         if (!new_devdata)
 723                 return -ENOMEM;
 724
 725         spin_lock_irqsave(&devdata_mutex, flags);
 726         old_devdata = rcu_dereference_check(devdata,
 727                         lockdep_is_held(&devdata_mutex));
 728         if (old_devdata)
 729                 of_node = old_devdata->dev->of_node;
 730
 731         if (!old_devdata || !of_node) {
 732                 pr_err("%s: device is not available\n", __func__);
 733                 spin_unlock_irqrestore(&devdata_mutex, flags);
 734                 kfree(new_devdata);
 735                 return -ENODEV;
 736         }
 737
 738         memcpy(new_devdata, old_devdata, sizeof(*old_devdata));
 739         new_devdata->counters = old_devdata->counters;
 740
 741         /* Set ptrs for existing properties */
 742         status = of_find_property(of_node, "status", NULL);
 743         maxsglen = of_find_property(of_node, "ibm,max-sg-len", NULL);
 744         maxsyncop = of_find_property(of_node, "ibm,max-sync-cop", NULL);
 745         if (!status || !maxsglen || !maxsyncop) {
 746                 dev_err(old_devdata->dev, "%s: Could not locate device properties\n", __func__);
 747                 ret = -EINVAL;
 748                 goto error_out;
 749         }
 750
 751         /*
 752          * If this is a property update, there are only certain properties that
 753          * we care about. Bail if it isn't in the below list
 754          */
 755         if (new_prop && (strncmp(new_prop->name, "status", new_prop->length) ||
 756                          strncmp(new_prop->name, "ibm,max-sg-len", new_prop->length) ||
 757                          strncmp(new_prop->name, "ibm,max-sync-cop", new_prop->length)))
 758                 goto out;
 759
 760         /* Perform property updates */
 761         ret = nx842_OF_upd_status(status);
 762         if (ret)
 763                 goto error_out;
 764
 765         ret = nx842_OF_upd_maxsglen(new_devdata, maxsglen);
 766         if (ret)
 767                 goto error_out;
 768
 769         ret = nx842_OF_upd_maxsyncop(new_devdata, maxsyncop);
 770         if (ret)
 771                 goto error_out;
 772
 773 out:
 774         dev_info(old_devdata->dev, "%s: max_sync_size new:%u old:%u\n",
 775                         __func__, new_devdata->max_sync_size,
 776                         old_devdata->max_sync_size);
 777         dev_info(old_devdata->dev, "%s: max_sync_sg new:%u old:%u\n",
 778                         __func__, new_devdata->max_sync_sg,
 779                         old_devdata->max_sync_sg);
 780         dev_info(old_devdata->dev, "%s: max_sg_len new:%u old:%u\n",
 781                         __func__, new_devdata->max_sg_len,
 782                         old_devdata->max_sg_len);
 783
 784         rcu_assign_pointer(devdata, new_devdata);
 785         spin_unlock_irqrestore(&devdata_mutex, flags);
 786         synchronize_rcu();
 787         dev_set_drvdata(new_devdata->dev, new_devdata);
 788         kfree(old_devdata);
 789         return 0;
 790
 791 error_out:
 792         if (new_devdata) {
 793                 dev_info(old_devdata->dev, "%s: device disabled\n", __func__);
 794                 nx842_OF_set_defaults(new_devdata);
 795                 rcu_assign_pointer(devdata, new_devdata);
 796                 spin_unlock_irqrestore(&devdata_mutex, flags);
 797                 synchronize_rcu();
 798                 dev_set_drvdata(new_devdata->dev, new_devdata);
 799                 kfree(old_devdata);
 800         } else {
 801                 dev_err(old_devdata->dev, "%s: could not update driver from hardware\n", __func__);
 802                 spin_unlock_irqrestore(&devdata_mutex, flags);
 803         }
 804
 805         if (!ret)
 806                 ret = -EINVAL;
 807         return ret;
 808 }
 809
 810 /**
 811  * nx842_OF_notifier - Process updates to OF properties for the device
 812  *
 813  * @np: notifier block
 814  * @action: notifier action
 815  * @update: struct pSeries_reconfig_prop_update pointer if action is
 816  *      PSERIES_UPDATE_PROPERTY
 817  *
 818  * Returns:
 819  *      NOTIFY_OK on success
 820  *      NOTIFY_BAD encoded with error number on failure, use
 821  *              notifier_to_errno() to decode this value
 822  */
 823 static int nx842_OF_notifier(struct notifier_block *np, unsigned long action,
 824                              void *data)
 825 {
 826         struct of_reconfig_data *upd = data;
 827         struct nx842_devdata *local_devdata;
 828         struct device_node *node = NULL;
 829
 830         rcu_read_lock();
 831         local_devdata = rcu_dereference(devdata);
 832         if (local_devdata)
 833                 node = local_devdata->dev->of_node;
 834
 835         if (local_devdata &&
 836                         action == OF_RECONFIG_UPDATE_PROPERTY &&
 837                         !strcmp(upd->dn->name, node->name)) {
 838                 rcu_read_unlock();
 839                 nx842_OF_upd(upd->prop);
 840         } else
 841                 rcu_read_unlock();
 842
 843         return NOTIFY_OK;
 844 }
 845
 846 static struct notifier_block nx842_of_nb = {
 847         .notifier_call = nx842_OF_notifier,
 848 };
 849
 850 #define nx842_counter_read(_name)                                       \
 851 static ssize_t nx842_##_name##_show(struct device *dev,         \
 852                 struct device_attribute *attr,                          \
 853                 char *buf) {                                            \
 854         struct nx842_devdata *local_devdata;                    \
 855         int p = 0;                                                      \
 856         rcu_read_lock();                                                \
 857         local_devdata = rcu_dereference(devdata);                       \
 858         if (local_devdata)                                              \
 859                 p = snprintf(buf, PAGE_SIZE, "%lld\n",                  \
 860                        atomic64_read(&local_devdata->counters->_name)); \
 861         rcu_read_unlock();                                              \
 862         return p;                                                       \
 863 }
 864
 865 #define NX842DEV_COUNTER_ATTR_RO(_name)                                 \
 866         nx842_counter_read(_name);                                      \
 867         static struct device_attribute dev_attr_##_name = __ATTR(_name, \
 868                                                 0444,                   \
 869                                                 nx842_##_name##_show,\
 870                                                 NULL);
 871
 872 NX842DEV_COUNTER_ATTR_RO(comp_complete);
 873 NX842DEV_COUNTER_ATTR_RO(comp_failed);
 874 NX842DEV_COUNTER_ATTR_RO(decomp_complete);
 875 NX842DEV_COUNTER_ATTR_RO(decomp_failed);
 876 NX842DEV_COUNTER_ATTR_RO(swdecomp);
 877
 878 static ssize_t nx842_timehist_show(struct device *,
 879                 struct device_attribute *, char *);
 880
 881 static struct device_attribute dev_attr_comp_times = __ATTR(comp_times, 0444,
 882                 nx842_timehist_show, NULL);
 883 static struct device_attribute dev_attr_decomp_times = __ATTR(decomp_times,
 884                 0444, nx842_timehist_show, NULL);
 885
 886 static ssize_t nx842_timehist_show(struct device *dev,
 887                 struct device_attribute *attr, char *buf) {
 888         char *p = buf;
 889         struct nx842_devdata *local_devdata;
 890         atomic64_t *times;
 891         int bytes_remain = PAGE_SIZE;
 892         int bytes;
 893         int i;
 894
 895         rcu_read_lock();
 896         local_devdata = rcu_dereference(devdata);
 897         if (!local_devdata) {
 898                 rcu_read_unlock();
 899                 return 0;
 900         }
 901
 902         if (attr == &dev_attr_comp_times)
 903                 times = local_devdata->counters->comp_times;
 904         else if (attr == &dev_attr_decomp_times)
 905                 times = local_devdata->counters->decomp_times;
 906         else {
 907                 rcu_read_unlock();
 908                 return 0;
 909         }
 910
 911         for (i = 0; i < (NX842_HIST_SLOTS - 2); i++) {
 912                 bytes = snprintf(p, bytes_remain, "%u-%uus:\t%lld\n",
 913                                i ? (2<<(i-1)) : 0, (2<<i)-1,
 914                                atomic64_read(&times[i]));
 915                 bytes_remain -= bytes;
 916                 p += bytes;
 917         }
 918         /* The last bucket holds everything over
 919          * 2<<(NX842_HIST_SLOTS - 2) us */
 920         bytes = snprintf(p, bytes_remain, "%uus - :\t%lld\n",
 921                         2<<(NX842_HIST_SLOTS - 2),
 922                         atomic64_read(&times[(NX842_HIST_SLOTS - 1)]));
 923         p += bytes;
 924
 925         rcu_read_unlock();
 926         return p - buf;
 927 }
 928
 929 static struct attribute *nx842_sysfs_entries[] = {
 930         &dev_attr_comp_complete.attr,
 931         &dev_attr_comp_failed.attr,
 932         &dev_attr_decomp_complete.attr,
 933         &dev_attr_decomp_failed.attr,
 934         &dev_attr_swdecomp.attr,
 935         &dev_attr_comp_times.attr,
 936         &dev_attr_decomp_times.attr,
 937         NULL,
 938 };
 939
 940 static struct attribute_group nx842_attribute_group = {
 941         .name = NULL,           /* put in device directory */
 942         .attrs = nx842_sysfs_entries,
 943 };
 944
 945 static struct nx842_driver nx842_pseries_driver = {
 946         .name =         KBUILD_MODNAME,
 947         .owner =        THIS_MODULE,
 948         .workmem_size = sizeof(struct nx842_workmem),
 949         .constraints =  &nx842_pseries_constraints,
 950         .compress =     nx842_pseries_compress,
 951         .decompress =   nx842_pseries_decompress,
 952 };
 953
 954 static int nx842_pseries_crypto_init(struct crypto_tfm *tfm)
 955 {
 956         return nx842_crypto_init(tfm, &nx842_pseries_driver);
 957 }
 958
 959 static struct crypto_alg nx842_pseries_alg = {
 960         .cra_name               = "842",
 961         .cra_driver_name        = "842-nx",
 962         .cra_priority           = 300,
 963         .cra_flags              = CRYPTO_ALG_TYPE_COMPRESS,
 964         .cra_ctxsize            = sizeof(struct nx842_crypto_ctx),
 965         .cra_module             = THIS_MODULE,
 966         .cra_init               = nx842_pseries_crypto_init,
 967         .cra_exit               = nx842_crypto_exit,
 968         .cra_u                  = { .compress = {
 969         .coa_compress           = nx842_crypto_compress,
 970         .coa_decompress         = nx842_crypto_decompress } }
 971 };
 972
 973 static int nx842_probe(struct vio_dev *viodev,
 974                        const struct vio_device_id *id)
 975 {
 976         struct nx842_devdata *old_devdata, *new_devdata = NULL;
 977         unsigned long flags;
 978         int ret = 0;
 979
 980         new_devdata = kzalloc(sizeof(*new_devdata), GFP_NOFS);
 981         if (!new_devdata)
 982                 return -ENOMEM;
 983
 984         new_devdata->counters = kzalloc(sizeof(*new_devdata->counters),
 985                         GFP_NOFS);
 986         if (!new_devdata->counters) {
 987                 kfree(new_devdata);
 988                 return -ENOMEM;
 989         }
 990
 991         spin_lock_irqsave(&devdata_mutex, flags);
 992         old_devdata = rcu_dereference_check(devdata,
 993                         lockdep_is_held(&devdata_mutex));
 994
 995         if (old_devdata && old_devdata->vdev != NULL) {
 996                 dev_err(&viodev->dev, "%s: Attempt to register more than one instance of the hardware\n", __func__);
 997                 ret = -1;
 998                 goto error_unlock;
 999         }
1000
1001         dev_set_drvdata(&viodev->dev, NULL);
1002
1003         new_devdata->vdev = viodev;
1004         new_devdata->dev = &viodev->dev;
1005         nx842_OF_set_defaults(new_devdata);
1006
1007         rcu_assign_pointer(devdata, new_devdata);
1008         spin_unlock_irqrestore(&devdata_mutex, flags);
1009         synchronize_rcu();
1010         kfree(old_devdata);
1011
1012         of_reconfig_notifier_register(&nx842_of_nb);
1013
1014         ret = nx842_OF_upd(NULL);
1015         if (ret)
1016                 goto error;
1017
1018         ret = crypto_register_alg(&nx842_pseries_alg);
1019         if (ret) {
1020                 dev_err(&viodev->dev, "could not register comp alg: %d\n", ret);
1021                 goto error;
1022         }
1023
1024         rcu_read_lock();
1025         dev_set_drvdata(&viodev->dev, rcu_dereference(devdata));
1026         rcu_read_unlock();
1027
1028         if (sysfs_create_group(&viodev->dev.kobj, &nx842_attribute_group)) {
1029                 dev_err(&viodev->dev, "could not create sysfs device attributes\n");
1030                 ret = -1;
1031                 goto error;
1032         }
1033
1034         return 0;
1035
1036 error_unlock:
1037         spin_unlock_irqrestore(&devdata_mutex, flags);
1038         if (new_devdata)
1039                 kfree(new_devdata->counters);
1040         kfree(new_devdata);
1041 error:
1042         return ret;
1043 }
1044
1045 static int nx842_remove(struct vio_dev *viodev)
1046 {
1047         struct nx842_devdata *old_devdata;
1048         unsigned long flags;
1049
1050         pr_info("Removing IBM Power 842 compression device\n");
1051         sysfs_remove_group(&viodev->dev.kobj, &nx842_attribute_group);
1052
1053         crypto_unregister_alg(&nx842_pseries_alg);
1054
1055         spin_lock_irqsave(&devdata_mutex, flags);
1056         old_devdata = rcu_dereference_check(devdata,
1057                         lockdep_is_held(&devdata_mutex));
1058         of_reconfig_notifier_unregister(&nx842_of_nb);
1059         RCU_INIT_POINTER(devdata, NULL);
1060         spin_unlock_irqrestore(&devdata_mutex, flags);
1061         synchronize_rcu();
1062         dev_set_drvdata(&viodev->dev, NULL);
1063         if (old_devdata)
1064                 kfree(old_devdata->counters);
1065         kfree(old_devdata);
1066
1067         return 0;
1068 }
1069
1070 static const struct vio_device_id nx842_vio_driver_ids[] = {
1071         {"ibm,compression-v1", "ibm,compression"},
1072         {"", ""},
1073 };
1074
1075 static struct vio_driver nx842_vio_driver = {
1076         .name = KBUILD_MODNAME,
1077         .probe = nx842_probe,
1078         .remove = nx842_remove,
1079         .get_desired_dma = nx842_get_desired_dma,
1080         .id_table = nx842_vio_driver_ids,
1081 };
1082
1083 static int __init nx842_pseries_init(void)
1084 {
1085         struct nx842_devdata *new_devdata;
1086         int ret;
1087
1088         if (!of_find_compatible_node(NULL, NULL, "ibm,compression"))
1089                 return -ENODEV;
1090
1091         RCU_INIT_POINTER(devdata, NULL);
1092         new_devdata = kzalloc(sizeof(*new_devdata), GFP_KERNEL);
1093         if (!new_devdata)
1094                 return -ENOMEM;
1095
1096         RCU_INIT_POINTER(devdata, new_devdata);
1097
1098         ret = vio_register_driver(&nx842_vio_driver);
1099         if (ret) {
1100                 pr_err("Could not register VIO driver %d\n", ret);
1101
1102                 kfree(new_devdata);
1103                 return ret;
1104         }
1105
1106         return 0;
1107 }
1108
1109 module_init(nx842_pseries_init);
1110
1111 static void __exit nx842_pseries_exit(void)
1112 {
1113         struct nx842_devdata *old_devdata;
1114         unsigned long flags;
1115
1116         crypto_unregister_alg(&nx842_pseries_alg);
1117
1118         spin_lock_irqsave(&devdata_mutex, flags);
1119         old_devdata = rcu_dereference_check(devdata,
1120                         lockdep_is_held(&devdata_mutex));
1121         RCU_INIT_POINTER(devdata, NULL);
1122         spin_unlock_irqrestore(&devdata_mutex, flags);
1123         synchronize_rcu();
1124         if (old_devdata && old_devdata->dev)
1125                 dev_set_drvdata(old_devdata->dev, NULL);
1126         kfree(old_devdata);
1127         vio_unregister_driver(&nx842_vio_driver);
1128 }
1129
1130 module_exit(nx842_pseries_exit);
1131