drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c

   1 /*
   2  * Copyright 2016 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Author: Huang Rui
  23  *
  24  */
  25
  26 #include <linux/firmware.h>
  27
  28 #include "amdgpu.h"
  29 #include "amdgpu_psp.h"
  30 #include "amdgpu_ucode.h"
  31 #include "soc15_common.h"
  32 #include "psp_v3_1.h"
  33 #include "psp_v10_0.h"
  34 #include "psp_v11_0.h"
  35 #include "psp_v12_0.h"
  36
  37 static void psp_set_funcs(struct amdgpu_device *adev);
  38
  39 static int psp_early_init(void *handle)
  40 {
  41         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
  42         struct psp_context *psp = &adev->psp;
  43
  44         psp_set_funcs(adev);
  45
  46         switch (adev->asic_type) {
  47         case CHIP_VEGA10:
  48         case CHIP_VEGA12:
  49                 psp_v3_1_set_psp_funcs(psp);
  50                 psp->autoload_supported = false;
  51                 break;
  52         case CHIP_RAVEN:
  53                 psp_v10_0_set_psp_funcs(psp);
  54                 psp->autoload_supported = false;
  55                 break;
  56         case CHIP_VEGA20:
  57         case CHIP_ARCTURUS:
  58                 psp_v11_0_set_psp_funcs(psp);
  59                 psp->autoload_supported = false;
  60                 break;
  61         case CHIP_NAVI10:
  62         case CHIP_NAVI14:
  63         case CHIP_NAVI12:
  64                 psp_v11_0_set_psp_funcs(psp);
  65                 psp->autoload_supported = true;
  66                 break;
  67         case CHIP_RENOIR:
  68                 psp_v12_0_set_psp_funcs(psp);
  69                 break;
  70         default:
  71                 return -EINVAL;
  72         }
  73
  74         psp->adev = adev;
  75
  76         return 0;
  77 }
  78
  79 static int psp_sw_init(void *handle)
  80 {
  81         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
  82         struct psp_context *psp = &adev->psp;
  83         int ret;
  84
  85         ret = psp_init_microcode(psp);
  86         if (ret) {
  87                 DRM_ERROR("Failed to load psp firmware!\n");
  88                 return ret;
  89         }
  90
  91         return 0;
  92 }
  93
  94 static int psp_sw_fini(void *handle)
  95 {
  96         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
  97
  98         release_firmware(adev->psp.sos_fw);
  99         adev->psp.sos_fw = NULL;
 100         release_firmware(adev->psp.asd_fw);
 101         adev->psp.asd_fw = NULL;
 102         if (adev->psp.ta_fw) {
 103                 release_firmware(adev->psp.ta_fw);
 104                 adev->psp.ta_fw = NULL;
 105         }
 106         return 0;
 107 }
 108
 109 int psp_wait_for(struct psp_context *psp, uint32_t reg_index,
 110                  uint32_t reg_val, uint32_t mask, bool check_changed)
 111 {
 112         uint32_t val;
 113         int i;
 114         struct amdgpu_device *adev = psp->adev;
 115
 116         for (i = 0; i < adev->usec_timeout; i++) {
 117                 val = RREG32(reg_index);
 118                 if (check_changed) {
 119                         if (val != reg_val)
 120                                 return 0;
 121                 } else {
 122                         if ((val & mask) == reg_val)
 123                                 return 0;
 124                 }
 125                 udelay(1);
 126         }
 127
 128         return -ETIME;
 129 }
 130
 131 static int
 132 psp_cmd_submit_buf(struct psp_context *psp,
 133                    struct amdgpu_firmware_info *ucode,
 134                    struct psp_gfx_cmd_resp *cmd, uint64_t fence_mc_addr)
 135 {
 136         int ret;
 137         int index;
 138         int timeout = 2000;
 139
 140         mutex_lock(&psp->mutex);
 141
 142         memset(psp->cmd_buf_mem, 0, PSP_CMD_BUFFER_SIZE);
 143
 144         memcpy(psp->cmd_buf_mem, cmd, sizeof(struct psp_gfx_cmd_resp));
 145
 146         index = atomic_inc_return(&psp->fence_value);
 147         ret = psp_cmd_submit(psp, psp->cmd_buf_mc_addr, fence_mc_addr, index);
 148         if (ret) {
 149                 atomic_dec(&psp->fence_value);
 150                 mutex_unlock(&psp->mutex);
 151                 return ret;
 152         }
 153
 154         while (*((unsigned int *)psp->fence_buf) != index) {
 155                 if (--timeout == 0)
 156                         break;
 157                 msleep(1);
 158         }
 159
 160         /* In some cases, psp response status is not 0 even there is no
 161          * problem while the command is submitted. Some version of PSP FW
 162          * doesn't write 0 to that field.
 163          * So here we would like to only print a warning instead of an error
 164          * during psp initialization to avoid breaking hw_init and it doesn't
 165          * return -EINVAL.
 166          */
 167         if (psp->cmd_buf_mem->resp.status || !timeout) {
 168                 if (ucode)
 169                         DRM_WARN("failed to load ucode id (%d) ",
 170                                   ucode->ucode_id);
 171                 DRM_WARN("psp command failed and response status is (0x%X)\n",
 172                           psp->cmd_buf_mem->resp.status & GFX_CMD_STATUS_MASK);
 173                 if (!timeout) {
 174                         mutex_unlock(&psp->mutex);
 175                         return -EINVAL;
 176                 }
 177         }
 178
 179         /* get xGMI session id from response buffer */
 180         cmd->resp.session_id = psp->cmd_buf_mem->resp.session_id;
 181
 182         if (ucode) {
 183                 ucode->tmr_mc_addr_lo = psp->cmd_buf_mem->resp.fw_addr_lo;
 184                 ucode->tmr_mc_addr_hi = psp->cmd_buf_mem->resp.fw_addr_hi;
 185         }
 186         mutex_unlock(&psp->mutex);
 187
 188         return ret;
 189 }
 190
 191 static void psp_prep_tmr_cmd_buf(struct psp_context *psp,
 192                                  struct psp_gfx_cmd_resp *cmd,
 193                                  uint64_t tmr_mc, uint32_t size)
 194 {
 195         if (psp_support_vmr_ring(psp))
 196                 cmd->cmd_id = GFX_CMD_ID_SETUP_VMR;
 197         else
 198                 cmd->cmd_id = GFX_CMD_ID_SETUP_TMR;
 199         cmd->cmd.cmd_setup_tmr.buf_phy_addr_lo = lower_32_bits(tmr_mc);
 200         cmd->cmd.cmd_setup_tmr.buf_phy_addr_hi = upper_32_bits(tmr_mc);
 201         cmd->cmd.cmd_setup_tmr.buf_size = size;
 202 }
 203
 204 static void psp_prep_load_toc_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 205                                       uint64_t pri_buf_mc, uint32_t size)
 206 {
 207         cmd->cmd_id = GFX_CMD_ID_LOAD_TOC;
 208         cmd->cmd.cmd_load_toc.toc_phy_addr_lo = lower_32_bits(pri_buf_mc);
 209         cmd->cmd.cmd_load_toc.toc_phy_addr_hi = upper_32_bits(pri_buf_mc);
 210         cmd->cmd.cmd_load_toc.toc_size = size;
 211 }
 212
 213 /* Issue LOAD TOC cmd to PSP to part toc and calculate tmr size needed */
 214 static int psp_load_toc(struct psp_context *psp,
 215                         uint32_t *tmr_size)
 216 {
 217         int ret;
 218         struct psp_gfx_cmd_resp *cmd;
 219
 220         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 221         if (!cmd)
 222                 return -ENOMEM;
 223         /* Copy toc to psp firmware private buffer */
 224         memset(psp->fw_pri_buf, 0, PSP_1_MEG);
 225         memcpy(psp->fw_pri_buf, psp->toc_start_addr, psp->toc_bin_size);
 226
 227         psp_prep_load_toc_cmd_buf(cmd, psp->fw_pri_mc_addr, psp->toc_bin_size);
 228
 229         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 230                                  psp->fence_buf_mc_addr);
 231         if (!ret)
 232                 *tmr_size = psp->cmd_buf_mem->resp.tmr_size;
 233         kfree(cmd);
 234         return ret;
 235 }
 236
 237 /* Set up Trusted Memory Region */
 238 static int psp_tmr_init(struct psp_context *psp)
 239 {
 240         int ret;
 241         int tmr_size;
 242         void *tmr_buf;
 243         void **pptr;
 244
 245         /*
 246          * According to HW engineer, they prefer the TMR address be "naturally
 247          * aligned" , e.g. the start address be an integer divide of TMR size.
 248          *
 249          * Note: this memory need be reserved till the driver
 250          * uninitializes.
 251          */
 252         tmr_size = PSP_TMR_SIZE;
 253
 254         /* For ASICs support RLC autoload, psp will parse the toc
 255          * and calculate the total size of TMR needed */
 256         if (psp->toc_start_addr &&
 257             psp->toc_bin_size &&
 258             psp->fw_pri_buf) {
 259                 ret = psp_load_toc(psp, &tmr_size);
 260                 if (ret) {
 261                         DRM_ERROR("Failed to load toc\n");
 262                         return ret;
 263                 }
 264         }
 265
 266         pptr = amdgpu_sriov_vf(psp->adev) ? &tmr_buf : NULL;
 267         ret = amdgpu_bo_create_kernel(psp->adev, tmr_size, PSP_TMR_SIZE,
 268                                       AMDGPU_GEM_DOMAIN_VRAM,
 269                                       &psp->tmr_bo, &psp->tmr_mc_addr, pptr);
 270
 271         return ret;
 272 }
 273
 274 static int psp_tmr_load(struct psp_context *psp)
 275 {
 276         int ret;
 277         struct psp_gfx_cmd_resp *cmd;
 278
 279         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 280         if (!cmd)
 281                 return -ENOMEM;
 282
 283         psp_prep_tmr_cmd_buf(psp, cmd, psp->tmr_mc_addr,
 284                              amdgpu_bo_size(psp->tmr_bo));
 285         DRM_INFO("reserve 0x%lx from 0x%llx for PSP TMR\n",
 286                  amdgpu_bo_size(psp->tmr_bo), psp->tmr_mc_addr);
 287
 288         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 289                                  psp->fence_buf_mc_addr);
 290         if (ret)
 291                 goto failed;
 292
 293         kfree(cmd);
 294
 295         return 0;
 296
 297 failed:
 298         kfree(cmd);
 299         return ret;
 300 }
 301
 302 static void psp_prep_asd_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 303                                  uint64_t asd_mc, uint64_t asd_mc_shared,
 304                                  uint32_t size, uint32_t shared_size)
 305 {
 306         cmd->cmd_id = GFX_CMD_ID_LOAD_ASD;
 307         cmd->cmd.cmd_load_ta.app_phy_addr_lo = lower_32_bits(asd_mc);
 308         cmd->cmd.cmd_load_ta.app_phy_addr_hi = upper_32_bits(asd_mc);
 309         cmd->cmd.cmd_load_ta.app_len = size;
 310
 311         cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_lo = lower_32_bits(asd_mc_shared);
 312         cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_hi = upper_32_bits(asd_mc_shared);
 313         cmd->cmd.cmd_load_ta.cmd_buf_len = shared_size;
 314 }
 315
 316 static int psp_asd_init(struct psp_context *psp)
 317 {
 318         int ret;
 319
 320         /*
 321          * Allocate 16k memory aligned to 4k from Frame Buffer (local
 322          * physical) for shared ASD <-> Driver
 323          */
 324         ret = amdgpu_bo_create_kernel(psp->adev, PSP_ASD_SHARED_MEM_SIZE,
 325                                       PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM,
 326                                       &psp->asd_shared_bo,
 327                                       &psp->asd_shared_mc_addr,
 328                                       &psp->asd_shared_buf);
 329
 330         return ret;
 331 }
 332
 333 static int psp_asd_load(struct psp_context *psp)
 334 {
 335         int ret;
 336         struct psp_gfx_cmd_resp *cmd;
 337
 338         /* If PSP version doesn't match ASD version, asd loading will be failed.
 339          * add workaround to bypass it for sriov now.
 340          * TODO: add version check to make it common
 341          */
 342         if (amdgpu_sriov_vf(psp->adev))
 343                 return 0;
 344
 345         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 346         if (!cmd)
 347                 return -ENOMEM;
 348
 349         memset(psp->fw_pri_buf, 0, PSP_1_MEG);
 350         memcpy(psp->fw_pri_buf, psp->asd_start_addr, psp->asd_ucode_size);
 351
 352         psp_prep_asd_cmd_buf(cmd, psp->fw_pri_mc_addr, psp->asd_shared_mc_addr,
 353                              psp->asd_ucode_size, PSP_ASD_SHARED_MEM_SIZE);
 354
 355         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 356                                  psp->fence_buf_mc_addr);
 357
 358         kfree(cmd);
 359
 360         return ret;
 361 }
 362
 363 static void psp_prep_reg_prog_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 364                 uint32_t id, uint32_t value)
 365 {
 366         cmd->cmd_id = GFX_CMD_ID_PROG_REG;
 367         cmd->cmd.cmd_setup_reg_prog.reg_value = value;
 368         cmd->cmd.cmd_setup_reg_prog.reg_id = id;
 369 }
 370
 371 int psp_reg_program(struct psp_context *psp, enum psp_reg_prog_id reg,
 372                 uint32_t value)
 373 {
 374         struct psp_gfx_cmd_resp *cmd = NULL;
 375         int ret = 0;
 376
 377         if (reg >= PSP_REG_LAST)
 378                 return -EINVAL;
 379
 380         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 381         if (!cmd)
 382                 return -ENOMEM;
 383
 384         psp_prep_reg_prog_cmd_buf(cmd, reg, value);
 385         ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr);
 386
 387         kfree(cmd);
 388         return ret;
 389 }
 390
 391 static void psp_prep_xgmi_ta_load_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 392                                           uint64_t xgmi_ta_mc, uint64_t xgmi_mc_shared,
 393                                           uint32_t xgmi_ta_size, uint32_t shared_size)
 394 {
 395         cmd->cmd_id = GFX_CMD_ID_LOAD_TA;
 396         cmd->cmd.cmd_load_ta.app_phy_addr_lo = lower_32_bits(xgmi_ta_mc);
 397         cmd->cmd.cmd_load_ta.app_phy_addr_hi = upper_32_bits(xgmi_ta_mc);
 398         cmd->cmd.cmd_load_ta.app_len = xgmi_ta_size;
 399
 400         cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_lo = lower_32_bits(xgmi_mc_shared);
 401         cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_hi = upper_32_bits(xgmi_mc_shared);
 402         cmd->cmd.cmd_load_ta.cmd_buf_len = shared_size;
 403 }
 404
 405 static int psp_xgmi_init_shared_buf(struct psp_context *psp)
 406 {
 407         int ret;
 408
 409         /*
 410          * Allocate 16k memory aligned to 4k from Frame Buffer (local
 411          * physical) for xgmi ta <-> Driver
 412          */
 413         ret = amdgpu_bo_create_kernel(psp->adev, PSP_XGMI_SHARED_MEM_SIZE,
 414                                       PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM,
 415                                       &psp->xgmi_context.xgmi_shared_bo,
 416                                       &psp->xgmi_context.xgmi_shared_mc_addr,
 417                                       &psp->xgmi_context.xgmi_shared_buf);
 418
 419         return ret;
 420 }
 421
 422 static int psp_xgmi_load(struct psp_context *psp)
 423 {
 424         int ret;
 425         struct psp_gfx_cmd_resp *cmd;
 426
 427         /*
 428          * TODO: bypass the loading in sriov for now
 429          */
 430         if (amdgpu_sriov_vf(psp->adev))
 431                 return 0;
 432
 433         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 434         if (!cmd)
 435                 return -ENOMEM;
 436
 437         memset(psp->fw_pri_buf, 0, PSP_1_MEG);
 438         memcpy(psp->fw_pri_buf, psp->ta_xgmi_start_addr, psp->ta_xgmi_ucode_size);
 439
 440         psp_prep_xgmi_ta_load_cmd_buf(cmd, psp->fw_pri_mc_addr,
 441                                       psp->xgmi_context.xgmi_shared_mc_addr,
 442                                       psp->ta_xgmi_ucode_size, PSP_XGMI_SHARED_MEM_SIZE);
 443
 444         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 445                                  psp->fence_buf_mc_addr);
 446
 447         if (!ret) {
 448                 psp->xgmi_context.initialized = 1;
 449                 psp->xgmi_context.session_id = cmd->resp.session_id;
 450         }
 451
 452         kfree(cmd);
 453
 454         return ret;
 455 }
 456
 457 static void psp_prep_xgmi_ta_unload_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 458                                             uint32_t xgmi_session_id)
 459 {
 460         cmd->cmd_id = GFX_CMD_ID_UNLOAD_TA;
 461         cmd->cmd.cmd_unload_ta.session_id = xgmi_session_id;
 462 }
 463
 464 static int psp_xgmi_unload(struct psp_context *psp)
 465 {
 466         int ret;
 467         struct psp_gfx_cmd_resp *cmd;
 468
 469         /*
 470          * TODO: bypass the unloading in sriov for now
 471          */
 472         if (amdgpu_sriov_vf(psp->adev))
 473                 return 0;
 474
 475         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 476         if (!cmd)
 477                 return -ENOMEM;
 478
 479         psp_prep_xgmi_ta_unload_cmd_buf(cmd, psp->xgmi_context.session_id);
 480
 481         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 482                                  psp->fence_buf_mc_addr);
 483
 484         kfree(cmd);
 485
 486         return ret;
 487 }
 488
 489 static void psp_prep_xgmi_ta_invoke_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 490                                             uint32_t ta_cmd_id,
 491                                             uint32_t xgmi_session_id)
 492 {
 493         cmd->cmd_id = GFX_CMD_ID_INVOKE_CMD;
 494         cmd->cmd.cmd_invoke_cmd.session_id = xgmi_session_id;
 495         cmd->cmd.cmd_invoke_cmd.ta_cmd_id = ta_cmd_id;
 496         /* Note: cmd_invoke_cmd.buf is not used for now */
 497 }
 498
 499 int psp_xgmi_invoke(struct psp_context *psp, uint32_t ta_cmd_id)
 500 {
 501         int ret;
 502         struct psp_gfx_cmd_resp *cmd;
 503
 504         /*
 505          * TODO: bypass the loading in sriov for now
 506         */
 507         if (amdgpu_sriov_vf(psp->adev))
 508                 return 0;
 509
 510         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 511         if (!cmd)
 512                 return -ENOMEM;
 513
 514         psp_prep_xgmi_ta_invoke_cmd_buf(cmd, ta_cmd_id,
 515                                         psp->xgmi_context.session_id);
 516
 517         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 518                                  psp->fence_buf_mc_addr);
 519
 520         kfree(cmd);
 521
 522         return ret;
 523 }
 524
 525 static int psp_xgmi_terminate(struct psp_context *psp)
 526 {
 527         int ret;
 528
 529         if (!psp->xgmi_context.initialized)
 530                 return 0;
 531
 532         ret = psp_xgmi_unload(psp);
 533         if (ret)
 534                 return ret;
 535
 536         psp->xgmi_context.initialized = 0;
 537
 538         /* free xgmi shared memory */
 539         amdgpu_bo_free_kernel(&psp->xgmi_context.xgmi_shared_bo,
 540                         &psp->xgmi_context.xgmi_shared_mc_addr,
 541                         &psp->xgmi_context.xgmi_shared_buf);
 542
 543         return 0;
 544 }
 545
 546 static int psp_xgmi_initialize(struct psp_context *psp)
 547 {
 548         struct ta_xgmi_shared_memory *xgmi_cmd;
 549         int ret;
 550
 551         if (!psp->adev->psp.ta_fw)
 552                 return -ENOENT;
 553
 554         if (!psp->xgmi_context.initialized) {
 555                 ret = psp_xgmi_init_shared_buf(psp);
 556                 if (ret)
 557                         return ret;
 558         }
 559
 560         /* Load XGMI TA */
 561         ret = psp_xgmi_load(psp);
 562         if (ret)
 563                 return ret;
 564
 565         /* Initialize XGMI session */
 566         xgmi_cmd = (struct ta_xgmi_shared_memory *)(psp->xgmi_context.xgmi_shared_buf);
 567         memset(xgmi_cmd, 0, sizeof(struct ta_xgmi_shared_memory));
 568         xgmi_cmd->cmd_id = TA_COMMAND_XGMI__INITIALIZE;
 569
 570         ret = psp_xgmi_invoke(psp, xgmi_cmd->cmd_id);
 571
 572         return ret;
 573 }
 574
 575 // ras begin
 576 static void psp_prep_ras_ta_load_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 577                 uint64_t ras_ta_mc, uint64_t ras_mc_shared,
 578                 uint32_t ras_ta_size, uint32_t shared_size)
 579 {
 580         cmd->cmd_id = GFX_CMD_ID_LOAD_TA;
 581         cmd->cmd.cmd_load_ta.app_phy_addr_lo = lower_32_bits(ras_ta_mc);
 582         cmd->cmd.cmd_load_ta.app_phy_addr_hi = upper_32_bits(ras_ta_mc);
 583         cmd->cmd.cmd_load_ta.app_len = ras_ta_size;
 584
 585         cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_lo = lower_32_bits(ras_mc_shared);
 586         cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_hi = upper_32_bits(ras_mc_shared);
 587         cmd->cmd.cmd_load_ta.cmd_buf_len = shared_size;
 588 }
 589
 590 static int psp_ras_init_shared_buf(struct psp_context *psp)
 591 {
 592         int ret;
 593
 594         /*
 595          * Allocate 16k memory aligned to 4k from Frame Buffer (local
 596          * physical) for ras ta <-> Driver
 597          */
 598         ret = amdgpu_bo_create_kernel(psp->adev, PSP_RAS_SHARED_MEM_SIZE,
 599                         PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM,
 600                         &psp->ras.ras_shared_bo,
 601                         &psp->ras.ras_shared_mc_addr,
 602                         &psp->ras.ras_shared_buf);
 603
 604         return ret;
 605 }
 606
 607 static int psp_ras_load(struct psp_context *psp)
 608 {
 609         int ret;
 610         struct psp_gfx_cmd_resp *cmd;
 611
 612         /*
 613          * TODO: bypass the loading in sriov for now
 614          */
 615         if (amdgpu_sriov_vf(psp->adev))
 616                 return 0;
 617
 618         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 619         if (!cmd)
 620                 return -ENOMEM;
 621
 622         memset(psp->fw_pri_buf, 0, PSP_1_MEG);
 623         memcpy(psp->fw_pri_buf, psp->ta_ras_start_addr, psp->ta_ras_ucode_size);
 624
 625         psp_prep_ras_ta_load_cmd_buf(cmd, psp->fw_pri_mc_addr,
 626                         psp->ras.ras_shared_mc_addr,
 627                         psp->ta_ras_ucode_size, PSP_RAS_SHARED_MEM_SIZE);
 628
 629         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 630                         psp->fence_buf_mc_addr);
 631
 632         if (!ret) {
 633                 psp->ras.ras_initialized = 1;
 634                 psp->ras.session_id = cmd->resp.session_id;
 635         }
 636
 637         kfree(cmd);
 638
 639         return ret;
 640 }
 641
 642 static void psp_prep_ras_ta_unload_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 643                                                 uint32_t ras_session_id)
 644 {
 645         cmd->cmd_id = GFX_CMD_ID_UNLOAD_TA;
 646         cmd->cmd.cmd_unload_ta.session_id = ras_session_id;
 647 }
 648
 649 static int psp_ras_unload(struct psp_context *psp)
 650 {
 651         int ret;
 652         struct psp_gfx_cmd_resp *cmd;
 653
 654         /*
 655          * TODO: bypass the unloading in sriov for now
 656          */
 657         if (amdgpu_sriov_vf(psp->adev))
 658                 return 0;
 659
 660         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 661         if (!cmd)
 662                 return -ENOMEM;
 663
 664         psp_prep_ras_ta_unload_cmd_buf(cmd, psp->ras.session_id);
 665
 666         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 667                         psp->fence_buf_mc_addr);
 668
 669         kfree(cmd);
 670
 671         return ret;
 672 }
 673
 674 static void psp_prep_ras_ta_invoke_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 675                 uint32_t ta_cmd_id,
 676                 uint32_t ras_session_id)
 677 {
 678         cmd->cmd_id = GFX_CMD_ID_INVOKE_CMD;
 679         cmd->cmd.cmd_invoke_cmd.session_id = ras_session_id;
 680         cmd->cmd.cmd_invoke_cmd.ta_cmd_id = ta_cmd_id;
 681         /* Note: cmd_invoke_cmd.buf is not used for now */
 682 }
 683
 684 int psp_ras_invoke(struct psp_context *psp, uint32_t ta_cmd_id)
 685 {
 686         int ret;
 687         struct psp_gfx_cmd_resp *cmd;
 688
 689         /*
 690          * TODO: bypass the loading in sriov for now
 691          */
 692         if (amdgpu_sriov_vf(psp->adev))
 693                 return 0;
 694
 695         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 696         if (!cmd)
 697                 return -ENOMEM;
 698
 699         psp_prep_ras_ta_invoke_cmd_buf(cmd, ta_cmd_id,
 700                         psp->ras.session_id);
 701
 702         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 703                         psp->fence_buf_mc_addr);
 704
 705         kfree(cmd);
 706
 707         return ret;
 708 }
 709
 710 int psp_ras_enable_features(struct psp_context *psp,
 711                 union ta_ras_cmd_input *info, bool enable)
 712 {
 713         struct ta_ras_shared_memory *ras_cmd;
 714         int ret;
 715
 716         if (!psp->ras.ras_initialized)
 717                 return -EINVAL;
 718
 719         ras_cmd = (struct ta_ras_shared_memory *)psp->ras.ras_shared_buf;
 720         memset(ras_cmd, 0, sizeof(struct ta_ras_shared_memory));
 721
 722         if (enable)
 723                 ras_cmd->cmd_id = TA_RAS_COMMAND__ENABLE_FEATURES;
 724         else
 725                 ras_cmd->cmd_id = TA_RAS_COMMAND__DISABLE_FEATURES;
 726
 727         ras_cmd->ras_in_message = *info;
 728
 729         ret = psp_ras_invoke(psp, ras_cmd->cmd_id);
 730         if (ret)
 731                 return -EINVAL;
 732
 733         return ras_cmd->ras_status;
 734 }
 735
 736 static int psp_ras_terminate(struct psp_context *psp)
 737 {
 738         int ret;
 739
 740         if (!psp->ras.ras_initialized)
 741                 return 0;
 742
 743         ret = psp_ras_unload(psp);
 744         if (ret)
 745                 return ret;
 746
 747         psp->ras.ras_initialized = 0;
 748
 749         /* free ras shared memory */
 750         amdgpu_bo_free_kernel(&psp->ras.ras_shared_bo,
 751                         &psp->ras.ras_shared_mc_addr,
 752                         &psp->ras.ras_shared_buf);
 753
 754         return 0;
 755 }
 756
 757 static int psp_ras_initialize(struct psp_context *psp)
 758 {
 759         int ret;
 760
 761         if (!psp->ras.ras_initialized) {
 762                 ret = psp_ras_init_shared_buf(psp);
 763                 if (ret)
 764                         return ret;
 765         }
 766
 767         ret = psp_ras_load(psp);
 768         if (ret)
 769                 return ret;
 770
 771         return 0;
 772 }
 773 // ras end
 774
 775 static int psp_hw_start(struct psp_context *psp)
 776 {
 777         struct amdgpu_device *adev = psp->adev;
 778         int ret;
 779
 780         if (!amdgpu_sriov_vf(adev) || !adev->in_gpu_reset) {
 781                 if (psp->kdb_bin_size &&
 782                     (psp->funcs->bootloader_load_kdb != NULL)) {
 783                         ret = psp_bootloader_load_kdb(psp);
 784                         if (ret) {
 785                                 DRM_ERROR("PSP load kdb failed!\n");
 786                                 return ret;
 787                         }
 788                 }
 789
 790                 ret = psp_bootloader_load_sysdrv(psp);
 791                 if (ret) {
 792                         DRM_ERROR("PSP load sysdrv failed!\n");
 793                         return ret;
 794                 }
 795
 796                 ret = psp_bootloader_load_sos(psp);
 797                 if (ret) {
 798                         DRM_ERROR("PSP load sos failed!\n");
 799                         return ret;
 800                 }
 801         }
 802
 803         ret = psp_ring_create(psp, PSP_RING_TYPE__KM);
 804         if (ret) {
 805                 DRM_ERROR("PSP create ring failed!\n");
 806                 return ret;
 807         }
 808
 809         ret = psp_tmr_init(psp);
 810         if (ret) {
 811                 DRM_ERROR("PSP tmr init failed!\n");
 812                 return ret;
 813         }
 814
 815         ret = psp_tmr_load(psp);
 816         if (ret) {
 817                 DRM_ERROR("PSP load tmr failed!\n");
 818                 return ret;
 819         }
 820
 821         ret = psp_asd_init(psp);
 822         if (ret) {
 823                 DRM_ERROR("PSP asd init failed!\n");
 824                 return ret;
 825         }
 826
 827         ret = psp_asd_load(psp);
 828         if (ret) {
 829                 DRM_ERROR("PSP load asd failed!\n");
 830                 return ret;
 831         }
 832
 833         if (adev->gmc.xgmi.num_physical_nodes > 1) {
 834                 ret = psp_xgmi_initialize(psp);
 835                 /* Warning the XGMI seesion initialize failure
 836                  * Instead of stop driver initialization
 837                  */
 838                 if (ret)
 839                         dev_err(psp->adev->dev,
 840                                 "XGMI: Failed to initialize XGMI session\n");
 841         }
 842
 843         if (psp->adev->psp.ta_fw) {
 844                 ret = psp_ras_initialize(psp);
 845                 if (ret)
 846                         dev_err(psp->adev->dev,
 847                                         "RAS: Failed to initialize RAS\n");
 848         }
 849
 850         return 0;
 851 }
 852
 853 static int psp_get_fw_type(struct amdgpu_firmware_info *ucode,
 854                            enum psp_gfx_fw_type *type)
 855 {
 856         switch (ucode->ucode_id) {
 857         case AMDGPU_UCODE_ID_SDMA0:
 858                 *type = GFX_FW_TYPE_SDMA0;
 859                 break;
 860         case AMDGPU_UCODE_ID_SDMA1:
 861                 *type = GFX_FW_TYPE_SDMA1;
 862                 break;
 863         case AMDGPU_UCODE_ID_SDMA2:
 864                 *type = GFX_FW_TYPE_SDMA2;
 865                 break;
 866         case AMDGPU_UCODE_ID_SDMA3:
 867                 *type = GFX_FW_TYPE_SDMA3;
 868                 break;
 869         case AMDGPU_UCODE_ID_SDMA4:
 870                 *type = GFX_FW_TYPE_SDMA4;
 871                 break;
 872         case AMDGPU_UCODE_ID_SDMA5:
 873                 *type = GFX_FW_TYPE_SDMA5;
 874                 break;
 875         case AMDGPU_UCODE_ID_SDMA6:
 876                 *type = GFX_FW_TYPE_SDMA6;
 877                 break;
 878         case AMDGPU_UCODE_ID_SDMA7:
 879                 *type = GFX_FW_TYPE_SDMA7;
 880                 break;
 881         case AMDGPU_UCODE_ID_CP_CE:
 882                 *type = GFX_FW_TYPE_CP_CE;
 883                 break;
 884         case AMDGPU_UCODE_ID_CP_PFP:
 885                 *type = GFX_FW_TYPE_CP_PFP;
 886                 break;
 887         case AMDGPU_UCODE_ID_CP_ME:
 888                 *type = GFX_FW_TYPE_CP_ME;
 889                 break;
 890         case AMDGPU_UCODE_ID_CP_MEC1:
 891                 *type = GFX_FW_TYPE_CP_MEC;
 892                 break;
 893         case AMDGPU_UCODE_ID_CP_MEC1_JT:
 894                 *type = GFX_FW_TYPE_CP_MEC_ME1;
 895                 break;
 896         case AMDGPU_UCODE_ID_CP_MEC2:
 897                 *type = GFX_FW_TYPE_CP_MEC;
 898                 break;
 899         case AMDGPU_UCODE_ID_CP_MEC2_JT:
 900                 *type = GFX_FW_TYPE_CP_MEC_ME2;
 901                 break;
 902         case AMDGPU_UCODE_ID_RLC_G:
 903                 *type = GFX_FW_TYPE_RLC_G;
 904                 break;
 905         case AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL:
 906                 *type = GFX_FW_TYPE_RLC_RESTORE_LIST_SRM_CNTL;
 907                 break;
 908         case AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM:
 909                 *type = GFX_FW_TYPE_RLC_RESTORE_LIST_GPM_MEM;
 910                 break;
 911         case AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM:
 912                 *type = GFX_FW_TYPE_RLC_RESTORE_LIST_SRM_MEM;
 913                 break;
 914         case AMDGPU_UCODE_ID_SMC:
 915                 *type = GFX_FW_TYPE_SMU;
 916                 break;
 917         case AMDGPU_UCODE_ID_UVD:
 918                 *type = GFX_FW_TYPE_UVD;
 919                 break;
 920         case AMDGPU_UCODE_ID_UVD1:
 921                 *type = GFX_FW_TYPE_UVD1;
 922                 break;
 923         case AMDGPU_UCODE_ID_VCE:
 924                 *type = GFX_FW_TYPE_VCE;
 925                 break;
 926         case AMDGPU_UCODE_ID_VCN:
 927                 *type = GFX_FW_TYPE_VCN;
 928                 break;
 929         case AMDGPU_UCODE_ID_DMCU_ERAM:
 930                 *type = GFX_FW_TYPE_DMCU_ERAM;
 931                 break;
 932         case AMDGPU_UCODE_ID_DMCU_INTV:
 933                 *type = GFX_FW_TYPE_DMCU_ISR;
 934                 break;
 935         case AMDGPU_UCODE_ID_VCN0_RAM:
 936                 *type = GFX_FW_TYPE_VCN0_RAM;
 937                 break;
 938         case AMDGPU_UCODE_ID_VCN1_RAM:
 939                 *type = GFX_FW_TYPE_VCN1_RAM;
 940                 break;
 941         case AMDGPU_UCODE_ID_MAXIMUM:
 942         default:
 943                 return -EINVAL;
 944         }
 945
 946         return 0;
 947 }
 948
 949 static void psp_print_fw_hdr(struct psp_context *psp,
 950                              struct amdgpu_firmware_info *ucode)
 951 {
 952         struct amdgpu_device *adev = psp->adev;
 953         struct common_firmware_header *hdr;
 954
 955         switch (ucode->ucode_id) {
 956         case AMDGPU_UCODE_ID_SDMA0:
 957         case AMDGPU_UCODE_ID_SDMA1:
 958         case AMDGPU_UCODE_ID_SDMA2:
 959         case AMDGPU_UCODE_ID_SDMA3:
 960         case AMDGPU_UCODE_ID_SDMA4:
 961         case AMDGPU_UCODE_ID_SDMA5:
 962         case AMDGPU_UCODE_ID_SDMA6:
 963         case AMDGPU_UCODE_ID_SDMA7:
 964                 hdr = (struct common_firmware_header *)
 965                         adev->sdma.instance[ucode->ucode_id - AMDGPU_UCODE_ID_SDMA0].fw->data;
 966                 amdgpu_ucode_print_sdma_hdr(hdr);
 967                 break;
 968         case AMDGPU_UCODE_ID_CP_CE:
 969                 hdr = (struct common_firmware_header *)adev->gfx.ce_fw->data;
 970                 amdgpu_ucode_print_gfx_hdr(hdr);
 971                 break;
 972         case AMDGPU_UCODE_ID_CP_PFP:
 973                 hdr = (struct common_firmware_header *)adev->gfx.pfp_fw->data;
 974                 amdgpu_ucode_print_gfx_hdr(hdr);
 975                 break;
 976         case AMDGPU_UCODE_ID_CP_ME:
 977                 hdr = (struct common_firmware_header *)adev->gfx.me_fw->data;
 978                 amdgpu_ucode_print_gfx_hdr(hdr);
 979                 break;
 980         case AMDGPU_UCODE_ID_CP_MEC1:
 981                 hdr = (struct common_firmware_header *)adev->gfx.mec_fw->data;
 982                 amdgpu_ucode_print_gfx_hdr(hdr);
 983                 break;
 984         case AMDGPU_UCODE_ID_RLC_G:
 985                 hdr = (struct common_firmware_header *)adev->gfx.rlc_fw->data;
 986                 amdgpu_ucode_print_rlc_hdr(hdr);
 987                 break;
 988         case AMDGPU_UCODE_ID_SMC:
 989                 hdr = (struct common_firmware_header *)adev->pm.fw->data;
 990                 amdgpu_ucode_print_smc_hdr(hdr);
 991                 break;
 992         default:
 993                 break;
 994         }
 995 }
 996
 997 static int psp_prep_load_ip_fw_cmd_buf(struct amdgpu_firmware_info *ucode,
 998                                        struct psp_gfx_cmd_resp *cmd)
 999 {
1000         int ret;
1001         uint64_t fw_mem_mc_addr = ucode->mc_addr;
1002
1003         memset(cmd, 0, sizeof(struct psp_gfx_cmd_resp));
1004
1005         cmd->cmd_id = GFX_CMD_ID_LOAD_IP_FW;
1006         cmd->cmd.cmd_load_ip_fw.fw_phy_addr_lo = lower_32_bits(fw_mem_mc_addr);
1007         cmd->cmd.cmd_load_ip_fw.fw_phy_addr_hi = upper_32_bits(fw_mem_mc_addr);
1008         cmd->cmd.cmd_load_ip_fw.fw_size = ucode->ucode_size;
1009
1010         ret = psp_get_fw_type(ucode, &cmd->cmd.cmd_load_ip_fw.fw_type);
1011         if (ret)
1012                 DRM_ERROR("Unknown firmware type\n");
1013
1014         return ret;
1015 }
1016
1017 static int psp_execute_np_fw_load(struct psp_context *psp,
1018                                struct amdgpu_firmware_info *ucode)
1019 {
1020         int ret = 0;
1021
1022         ret = psp_prep_load_ip_fw_cmd_buf(ucode, psp->cmd);
1023         if (ret)
1024                 return ret;
1025
1026         ret = psp_cmd_submit_buf(psp, ucode, psp->cmd,
1027                                  psp->fence_buf_mc_addr);
1028
1029         return ret;
1030 }
1031
1032 static int psp_np_fw_load(struct psp_context *psp)
1033 {
1034         int i, ret;
1035         struct amdgpu_firmware_info *ucode;
1036         struct amdgpu_device* adev = psp->adev;
1037
1038         if (psp->autoload_supported) {
1039                 ucode = &adev->firmware.ucode[AMDGPU_UCODE_ID_SMC];
1040                 if (!ucode->fw)
1041                         goto out;
1042
1043                 ret = psp_execute_np_fw_load(psp, ucode);
1044                 if (ret)
1045                         return ret;
1046         }
1047
1048 out:
1049         for (i = 0; i < adev->firmware.max_ucodes; i++) {
1050                 ucode = &adev->firmware.ucode[i];
1051                 if (!ucode->fw)
1052                         continue;
1053
1054                 if (ucode->ucode_id == AMDGPU_UCODE_ID_SMC &&
1055                     (psp_smu_reload_quirk(psp) || psp->autoload_supported))
1056                         continue;
1057
1058                 if (amdgpu_sriov_vf(adev) &&
1059                    (ucode->ucode_id == AMDGPU_UCODE_ID_SDMA0
1060                     || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA1
1061                     || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA2
1062                     || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA3
1063                     || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA4
1064                     || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA5
1065                     || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA6
1066                     || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA7
1067                     || ucode->ucode_id == AMDGPU_UCODE_ID_RLC_G))
1068                         /*skip ucode loading in SRIOV VF */
1069                         continue;
1070
1071                 if (psp->autoload_supported &&
1072                     (ucode->ucode_id == AMDGPU_UCODE_ID_CP_MEC1_JT ||
1073                      ucode->ucode_id == AMDGPU_UCODE_ID_CP_MEC2_JT))
1074                         /* skip mec JT when autoload is enabled */
1075                         continue;
1076                 /* Renoir only needs to load mec jump table one time */
1077                 if (adev->asic_type == CHIP_RENOIR &&
1078                     ucode->ucode_id == AMDGPU_UCODE_ID_CP_MEC2_JT)
1079                         continue;
1080
1081                 psp_print_fw_hdr(psp, ucode);
1082
1083                 ret = psp_execute_np_fw_load(psp, ucode);
1084                 if (ret)
1085                         return ret;
1086
1087                 /* Start rlc autoload after psp recieved all the gfx firmware */
1088                 if (ucode->ucode_id == AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM) {
1089                         ret = psp_rlc_autoload(psp);
1090                         if (ret) {
1091                                 DRM_ERROR("Failed to start rlc autoload\n");
1092                                 return ret;
1093                         }
1094                 }
1095 #if 0
1096                 /* check if firmware loaded sucessfully */
1097                 if (!amdgpu_psp_check_fw_loading_status(adev, i))
1098                         return -EINVAL;
1099 #endif
1100         }
1101
1102         return 0;
1103 }
1104
1105 static int psp_load_fw(struct amdgpu_device *adev)
1106 {
1107         int ret;
1108         struct psp_context *psp = &adev->psp;
1109
1110         if (amdgpu_sriov_vf(adev) && adev->in_gpu_reset) {
1111                 psp_ring_stop(psp, PSP_RING_TYPE__KM); /* should not destroy ring, only stop */
1112                 goto skip_memalloc;
1113         }
1114
1115         psp->cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
1116         if (!psp->cmd)
1117                 return -ENOMEM;
1118
1119         /* this fw pri bo is not used under SRIOV */
1120         if (!amdgpu_sriov_vf(psp->adev)) {
1121                 ret = amdgpu_bo_create_kernel(adev, PSP_1_MEG, PSP_1_MEG,
1122                                               AMDGPU_GEM_DOMAIN_GTT,
1123                                               &psp->fw_pri_bo,
1124                                               &psp->fw_pri_mc_addr,
1125                                               &psp->fw_pri_buf);
1126                 if (ret)
1127                         goto failed;
1128         }
1129
1130         ret = amdgpu_bo_create_kernel(adev, PSP_FENCE_BUFFER_SIZE, PAGE_SIZE,
1131                                         AMDGPU_GEM_DOMAIN_VRAM,
1132                                         &psp->fence_buf_bo,
1133                                         &psp->fence_buf_mc_addr,
1134                                         &psp->fence_buf);
1135         if (ret)
1136                 goto failed;
1137
1138         ret = amdgpu_bo_create_kernel(adev, PSP_CMD_BUFFER_SIZE, PAGE_SIZE,
1139                                       AMDGPU_GEM_DOMAIN_VRAM,
1140                                       &psp->cmd_buf_bo, &psp->cmd_buf_mc_addr,
1141                                       (void **)&psp->cmd_buf_mem);
1142         if (ret)
1143                 goto failed;
1144
1145         memset(psp->fence_buf, 0, PSP_FENCE_BUFFER_SIZE);
1146
1147         ret = psp_ring_init(psp, PSP_RING_TYPE__KM);
1148         if (ret) {
1149                 DRM_ERROR("PSP ring init failed!\n");
1150                 goto failed;
1151         }
1152
1153 skip_memalloc:
1154         ret = psp_hw_start(psp);
1155         if (ret)
1156                 goto failed;
1157
1158         ret = psp_np_fw_load(psp);
1159         if (ret)
1160                 goto failed;
1161
1162         return 0;
1163
1164 failed:
1165         /*
1166          * all cleanup jobs (xgmi terminate, ras terminate,
1167          * ring destroy, cmd/fence/fw buffers destory,
1168          * psp->cmd destory) are delayed to psp_hw_fini
1169          */
1170         return ret;
1171 }
1172
1173 static int psp_hw_init(void *handle)
1174 {
1175         int ret;
1176         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1177
1178         mutex_lock(&adev->firmware.mutex);
1179         /*
1180          * This sequence is just used on hw_init only once, no need on
1181          * resume.
1182          */
1183         ret = amdgpu_ucode_init_bo(adev);
1184         if (ret)
1185                 goto failed;
1186
1187         ret = psp_load_fw(adev);
1188         if (ret) {
1189                 DRM_ERROR("PSP firmware loading failed\n");
1190                 goto failed;
1191         }
1192
1193         mutex_unlock(&adev->firmware.mutex);
1194         return 0;
1195
1196 failed:
1197         adev->firmware.load_type = AMDGPU_FW_LOAD_DIRECT;
1198         mutex_unlock(&adev->firmware.mutex);
1199         return -EINVAL;
1200 }
1201
1202 static int psp_hw_fini(void *handle)
1203 {
1204         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1205         struct psp_context *psp = &adev->psp;
1206         void *tmr_buf;
1207         void **pptr;
1208
1209         if (adev->gmc.xgmi.num_physical_nodes > 1 &&
1210             psp->xgmi_context.initialized == 1)
1211                 psp_xgmi_terminate(psp);
1212
1213         if (psp->adev->psp.ta_fw)
1214                 psp_ras_terminate(psp);
1215
1216         psp_ring_destroy(psp, PSP_RING_TYPE__KM);
1217
1218         pptr = amdgpu_sriov_vf(psp->adev) ? &tmr_buf : NULL;
1219         amdgpu_bo_free_kernel(&psp->tmr_bo, &psp->tmr_mc_addr, pptr);
1220         amdgpu_bo_free_kernel(&psp->fw_pri_bo,
1221                               &psp->fw_pri_mc_addr, &psp->fw_pri_buf);
1222         amdgpu_bo_free_kernel(&psp->fence_buf_bo,
1223                               &psp->fence_buf_mc_addr, &psp->fence_buf);
1224         amdgpu_bo_free_kernel(&psp->asd_shared_bo, &psp->asd_shared_mc_addr,
1225                               &psp->asd_shared_buf);
1226         amdgpu_bo_free_kernel(&psp->cmd_buf_bo, &psp->cmd_buf_mc_addr,
1227                               (void **)&psp->cmd_buf_mem);
1228
1229         kfree(psp->cmd);
1230         psp->cmd = NULL;
1231
1232         return 0;
1233 }
1234
1235 static int psp_suspend(void *handle)
1236 {
1237         int ret;
1238         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1239         struct psp_context *psp = &adev->psp;
1240
1241         if (adev->gmc.xgmi.num_physical_nodes > 1 &&
1242             psp->xgmi_context.initialized == 1) {
1243                 ret = psp_xgmi_terminate(psp);
1244                 if (ret) {
1245                         DRM_ERROR("Failed to terminate xgmi ta\n");
1246                         return ret;
1247                 }
1248         }
1249
1250         if (psp->adev->psp.ta_fw) {
1251                 ret = psp_ras_terminate(psp);
1252                 if (ret) {
1253                         DRM_ERROR("Failed to terminate ras ta\n");
1254                         return ret;
1255                 }
1256         }
1257
1258         ret = psp_ring_stop(psp, PSP_RING_TYPE__KM);
1259         if (ret) {
1260                 DRM_ERROR("PSP ring stop failed\n");
1261                 return ret;
1262         }
1263
1264         return 0;
1265 }
1266
1267 static int psp_resume(void *handle)
1268 {
1269         int ret;
1270         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1271         struct psp_context *psp = &adev->psp;
1272
1273         DRM_INFO("PSP is resuming...\n");
1274
1275         mutex_lock(&adev->firmware.mutex);
1276
1277         ret = psp_hw_start(psp);
1278         if (ret)
1279                 goto failed;
1280
1281         ret = psp_np_fw_load(psp);
1282         if (ret)
1283                 goto failed;
1284
1285         mutex_unlock(&adev->firmware.mutex);
1286
1287         return 0;
1288
1289 failed:
1290         DRM_ERROR("PSP resume failed\n");
1291         mutex_unlock(&adev->firmware.mutex);
1292         return ret;
1293 }
1294
1295 int psp_gpu_reset(struct amdgpu_device *adev)
1296 {
1297         int ret;
1298
1299         if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP)
1300                 return 0;
1301
1302         mutex_lock(&adev->psp.mutex);
1303         ret = psp_mode1_reset(&adev->psp);
1304         mutex_unlock(&adev->psp.mutex);
1305
1306         return ret;
1307 }
1308
1309 int psp_rlc_autoload_start(struct psp_context *psp)
1310 {
1311         int ret;
1312         struct psp_gfx_cmd_resp *cmd;
1313
1314         if (amdgpu_sriov_vf(psp->adev))
1315                 return 0;
1316
1317         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
1318         if (!cmd)
1319                 return -ENOMEM;
1320
1321         cmd->cmd_id = GFX_CMD_ID_AUTOLOAD_RLC;
1322
1323         ret = psp_cmd_submit_buf(psp, NULL, cmd,
1324                                  psp->fence_buf_mc_addr);
1325         kfree(cmd);
1326         return ret;
1327 }
1328
1329 int psp_update_vcn_sram(struct amdgpu_device *adev, int inst_idx,
1330                         uint64_t cmd_gpu_addr, int cmd_size)
1331 {
1332         struct amdgpu_firmware_info ucode = {0};
1333
1334         ucode.ucode_id = inst_idx ? AMDGPU_UCODE_ID_VCN1_RAM :
1335                 AMDGPU_UCODE_ID_VCN0_RAM;
1336         ucode.mc_addr = cmd_gpu_addr;
1337         ucode.ucode_size = cmd_size;
1338
1339         return psp_execute_np_fw_load(&adev->psp, &ucode);
1340 }
1341
1342 static bool psp_check_fw_loading_status(struct amdgpu_device *adev,
1343                                         enum AMDGPU_UCODE_ID ucode_type)
1344 {
1345         struct amdgpu_firmware_info *ucode = NULL;
1346
1347         if (!adev->firmware.fw_size)
1348                 return false;
1349
1350         ucode = &adev->firmware.ucode[ucode_type];
1351         if (!ucode->fw || !ucode->ucode_size)
1352                 return false;
1353
1354         return psp_compare_sram_data(&adev->psp, ucode, ucode_type);
1355 }
1356
1357 static int psp_set_clockgating_state(void *handle,
1358                                      enum amd_clockgating_state state)
1359 {
1360         return 0;
1361 }
1362
1363 static int psp_set_powergating_state(void *handle,
1364                                      enum amd_powergating_state state)
1365 {
1366         return 0;
1367 }
1368
1369 const struct amd_ip_funcs psp_ip_funcs = {
1370         .name = "psp",
1371         .early_init = psp_early_init,
1372         .late_init = NULL,
1373         .sw_init = psp_sw_init,
1374         .sw_fini = psp_sw_fini,
1375         .hw_init = psp_hw_init,
1376         .hw_fini = psp_hw_fini,
1377         .suspend = psp_suspend,
1378         .resume = psp_resume,
1379         .is_idle = NULL,
1380         .check_soft_reset = NULL,
1381         .wait_for_idle = NULL,
1382         .soft_reset = NULL,
1383         .set_clockgating_state = psp_set_clockgating_state,
1384         .set_powergating_state = psp_set_powergating_state,
1385 };
1386
1387 static const struct amdgpu_psp_funcs psp_funcs = {
1388         .check_fw_loading_status = psp_check_fw_loading_status,
1389 };
1390
1391 static void psp_set_funcs(struct amdgpu_device *adev)
1392 {
1393         if (NULL == adev->firmware.funcs)
1394                 adev->firmware.funcs = &psp_funcs;
1395 }
1396
1397 const struct amdgpu_ip_block_version psp_v3_1_ip_block =
1398 {
1399         .type = AMD_IP_BLOCK_TYPE_PSP,
1400         .major = 3,
1401         .minor = 1,
1402         .rev = 0,
1403         .funcs = &psp_ip_funcs,
1404 };
1405
1406 const struct amdgpu_ip_block_version psp_v10_0_ip_block =
1407 {
1408         .type = AMD_IP_BLOCK_TYPE_PSP,
1409         .major = 10,
1410         .minor = 0,
1411         .rev = 0,
1412         .funcs = &psp_ip_funcs,
1413 };
1414
1415 const struct amdgpu_ip_block_version psp_v11_0_ip_block =
1416 {
1417         .type = AMD_IP_BLOCK_TYPE_PSP,
1418         .major = 11,
1419         .minor = 0,
1420         .rev = 0,
1421         .funcs = &psp_ip_funcs,
1422 };
1423
1424 const struct amdgpu_ip_block_version psp_v12_0_ip_block =
1425 {
1426         .type = AMD_IP_BLOCK_TYPE_PSP,
1427         .major = 12,
1428         .minor = 0,
1429         .rev = 0,
1430         .funcs = &psp_ip_funcs,
1431 };