drivers/gpu/drm/amd/amdgpu/vcn_v1_0.c

   1 /*
   2  * Copyright 2016 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  */
  23
  24 #include <linux/firmware.h>
  25
  26 #include "amdgpu.h"
  27 #include "amdgpu_vcn.h"
  28 #include "soc15.h"
  29 #include "soc15d.h"
  30 #include "soc15_common.h"
  31
  32 #include "vcn/vcn_1_0_offset.h"
  33 #include "vcn/vcn_1_0_sh_mask.h"
  34 #include "hdp/hdp_4_0_offset.h"
  35 #include "mmhub/mmhub_9_1_offset.h"
  36 #include "mmhub/mmhub_9_1_sh_mask.h"
  37
  38 #include "ivsrcid/vcn/irqsrcs_vcn_1_0.h"
  39
  40 #define mmUVD_RBC_XX_IB_REG_CHECK                               0x05ab
  41 #define mmUVD_RBC_XX_IB_REG_CHECK_BASE_IDX      1
  42 #define mmUVD_REG_XX_MASK                                                       0x05ac
  43 #define mmUVD_REG_XX_MASK_BASE_IDX                              1
  44
  45 static int vcn_v1_0_stop(struct amdgpu_device *adev);
  46 static void vcn_v1_0_set_dec_ring_funcs(struct amdgpu_device *adev);
  47 static void vcn_v1_0_set_enc_ring_funcs(struct amdgpu_device *adev);
  48 static void vcn_v1_0_set_jpeg_ring_funcs(struct amdgpu_device *adev);
  49 static void vcn_v1_0_set_irq_funcs(struct amdgpu_device *adev);
  50 static void vcn_v1_0_jpeg_ring_set_patch_ring(struct amdgpu_ring *ring, uint32_t ptr);
  51 static int vcn_v1_0_set_powergating_state(void *handle, enum amd_powergating_state state);
  52 static int vcn_v1_0_pause_dpg_mode(struct amdgpu_device *adev,
  53                                 struct dpg_pause_state *new_state);
  54
  55 /**
  56  * vcn_v1_0_early_init - set function pointers
  57  *
  58  * @handle: amdgpu_device pointer
  59  *
  60  * Set ring and irq function pointers
  61  */
  62 static int vcn_v1_0_early_init(void *handle)
  63 {
  64         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
  65
  66         adev->vcn.num_enc_rings = 2;
  67
  68         vcn_v1_0_set_dec_ring_funcs(adev);
  69         vcn_v1_0_set_enc_ring_funcs(adev);
  70         vcn_v1_0_set_jpeg_ring_funcs(adev);
  71         vcn_v1_0_set_irq_funcs(adev);
  72
  73         return 0;
  74 }
  75
  76 /**
  77  * vcn_v1_0_sw_init - sw init for VCN block
  78  *
  79  * @handle: amdgpu_device pointer
  80  *
  81  * Load firmware and sw initialization
  82  */
  83 static int vcn_v1_0_sw_init(void *handle)
  84 {
  85         struct amdgpu_ring *ring;
  86         int i, r;
  87         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
  88
  89         /* VCN DEC TRAP */
  90         r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VCN, VCN_1_0__SRCID__UVD_SYSTEM_MESSAGE_INTERRUPT, &adev->vcn.irq);
  91         if (r)
  92                 return r;
  93
  94         /* VCN ENC TRAP */
  95         for (i = 0; i < adev->vcn.num_enc_rings; ++i) {
  96                 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VCN, i + VCN_1_0__SRCID__UVD_ENC_GENERAL_PURPOSE,
  97                                         &adev->vcn.irq);
  98                 if (r)
  99                         return r;
 100         }
 101
 102         /* VCN JPEG TRAP */
 103         r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VCN, 126, &adev->vcn.irq);
 104         if (r)
 105                 return r;
 106
 107         r = amdgpu_vcn_sw_init(adev);
 108         if (r)
 109                 return r;
 110
 111         if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
 112                 const struct common_firmware_header *hdr;
 113                 hdr = (const struct common_firmware_header *)adev->vcn.fw->data;
 114                 adev->firmware.ucode[AMDGPU_UCODE_ID_VCN].ucode_id = AMDGPU_UCODE_ID_VCN;
 115                 adev->firmware.ucode[AMDGPU_UCODE_ID_VCN].fw = adev->vcn.fw;
 116                 adev->firmware.fw_size +=
 117                         ALIGN(le32_to_cpu(hdr->ucode_size_bytes), PAGE_SIZE);
 118                 DRM_INFO("PSP loading VCN firmware\n");
 119         }
 120
 121         r = amdgpu_vcn_resume(adev);
 122         if (r)
 123                 return r;
 124
 125         ring = &adev->vcn.ring_dec;
 126         sprintf(ring->name, "vcn_dec");
 127         r = amdgpu_ring_init(adev, ring, 512, &adev->vcn.irq, 0);
 128         if (r)
 129                 return r;
 130
 131         adev->vcn.internal.scratch9 = adev->vcn.external.scratch9 =
 132                 SOC15_REG_OFFSET(UVD, 0, mmUVD_SCRATCH9);
 133         adev->vcn.internal.data0 = adev->vcn.external.data0 =
 134                 SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_DATA0);
 135         adev->vcn.internal.data1 = adev->vcn.external.data1 =
 136                 SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_DATA1);
 137         adev->vcn.internal.cmd = adev->vcn.external.cmd =
 138                 SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_CMD);
 139         adev->vcn.internal.nop = adev->vcn.external.nop =
 140                 SOC15_REG_OFFSET(UVD, 0, mmUVD_NO_OP);
 141
 142         for (i = 0; i < adev->vcn.num_enc_rings; ++i) {
 143                 ring = &adev->vcn.ring_enc[i];
 144                 sprintf(ring->name, "vcn_enc%d", i);
 145                 r = amdgpu_ring_init(adev, ring, 512, &adev->vcn.irq, 0);
 146                 if (r)
 147                         return r;
 148         }
 149
 150         ring = &adev->vcn.ring_jpeg;
 151         sprintf(ring->name, "vcn_jpeg");
 152         r = amdgpu_ring_init(adev, ring, 512, &adev->vcn.irq, 0);
 153         if (r)
 154                 return r;
 155
 156         adev->vcn.pause_dpg_mode = vcn_v1_0_pause_dpg_mode;
 157         adev->vcn.internal.jpeg_pitch = adev->vcn.external.jpeg_pitch =
 158                 SOC15_REG_OFFSET(UVD, 0, mmUVD_JPEG_PITCH);
 159
 160         return 0;
 161 }
 162
 163 /**
 164  * vcn_v1_0_sw_fini - sw fini for VCN block
 165  *
 166  * @handle: amdgpu_device pointer
 167  *
 168  * VCN suspend and free up sw allocation
 169  */
 170 static int vcn_v1_0_sw_fini(void *handle)
 171 {
 172         int r;
 173         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 174
 175         r = amdgpu_vcn_suspend(adev);
 176         if (r)
 177                 return r;
 178
 179         r = amdgpu_vcn_sw_fini(adev);
 180
 181         return r;
 182 }
 183
 184 /**
 185  * vcn_v1_0_hw_init - start and test VCN block
 186  *
 187  * @handle: amdgpu_device pointer
 188  *
 189  * Initialize the hardware, boot up the VCPU and do some testing
 190  */
 191 static int vcn_v1_0_hw_init(void *handle)
 192 {
 193         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 194         struct amdgpu_ring *ring = &adev->vcn.ring_dec;
 195         int i, r;
 196
 197         r = amdgpu_ring_test_helper(ring);
 198         if (r)
 199                 goto done;
 200
 201         for (i = 0; i < adev->vcn.num_enc_rings; ++i) {
 202                 ring = &adev->vcn.ring_enc[i];
 203                 ring->sched.ready = true;
 204                 r = amdgpu_ring_test_helper(ring);
 205                 if (r)
 206                         goto done;
 207         }
 208
 209         ring = &adev->vcn.ring_jpeg;
 210         r = amdgpu_ring_test_helper(ring);
 211         if (r)
 212                 goto done;
 213
 214 done:
 215         if (!r)
 216                 DRM_INFO("VCN decode and encode initialized successfully(under %s).\n",
 217                         (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG)?"DPG Mode":"SPG Mode");
 218
 219         return r;
 220 }
 221
 222 /**
 223  * vcn_v1_0_hw_fini - stop the hardware block
 224  *
 225  * @handle: amdgpu_device pointer
 226  *
 227  * Stop the VCN block, mark ring as not ready any more
 228  */
 229 static int vcn_v1_0_hw_fini(void *handle)
 230 {
 231         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 232         struct amdgpu_ring *ring = &adev->vcn.ring_dec;
 233
 234         if ((adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) ||
 235                 RREG32_SOC15(VCN, 0, mmUVD_STATUS))
 236                 vcn_v1_0_set_powergating_state(adev, AMD_PG_STATE_GATE);
 237
 238         ring->sched.ready = false;
 239
 240         return 0;
 241 }
 242
 243 /**
 244  * vcn_v1_0_suspend - suspend VCN block
 245  *
 246  * @handle: amdgpu_device pointer
 247  *
 248  * HW fini and suspend VCN block
 249  */
 250 static int vcn_v1_0_suspend(void *handle)
 251 {
 252         int r;
 253         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 254
 255         r = vcn_v1_0_hw_fini(adev);
 256         if (r)
 257                 return r;
 258
 259         r = amdgpu_vcn_suspend(adev);
 260
 261         return r;
 262 }
 263
 264 /**
 265  * vcn_v1_0_resume - resume VCN block
 266  *
 267  * @handle: amdgpu_device pointer
 268  *
 269  * Resume firmware and hw init VCN block
 270  */
 271 static int vcn_v1_0_resume(void *handle)
 272 {
 273         int r;
 274         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 275
 276         r = amdgpu_vcn_resume(adev);
 277         if (r)
 278                 return r;
 279
 280         r = vcn_v1_0_hw_init(adev);
 281
 282         return r;
 283 }
 284
 285 /**
 286  * vcn_v1_0_mc_resume_spg_mode - memory controller programming
 287  *
 288  * @adev: amdgpu_device pointer
 289  *
 290  * Let the VCN memory controller know it's offsets
 291  */
 292 static void vcn_v1_0_mc_resume_spg_mode(struct amdgpu_device *adev)
 293 {
 294         uint32_t size = AMDGPU_GPU_PAGE_ALIGN(adev->vcn.fw->size + 4);
 295         uint32_t offset;
 296
 297         /* cache window 0: fw */
 298         if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
 299                 WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW,
 300                              (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN].tmr_mc_addr_lo));
 301                 WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH,
 302                              (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN].tmr_mc_addr_hi));
 303                 WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_OFFSET0, 0);
 304                 offset = 0;
 305         } else {
 306                 WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW,
 307                         lower_32_bits(adev->vcn.gpu_addr));
 308                 WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH,
 309                         upper_32_bits(adev->vcn.gpu_addr));
 310                 offset = size;
 311                 WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_OFFSET0,
 312                              AMDGPU_UVD_FIRMWARE_OFFSET >> 3);
 313         }
 314
 315         WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_SIZE0, size);
 316
 317         /* cache window 1: stack */
 318         WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW,
 319                      lower_32_bits(adev->vcn.gpu_addr + offset));
 320         WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH,
 321                      upper_32_bits(adev->vcn.gpu_addr + offset));
 322         WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_OFFSET1, 0);
 323         WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_SIZE1, AMDGPU_VCN_STACK_SIZE);
 324
 325         /* cache window 2: context */
 326         WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW,
 327                      lower_32_bits(adev->vcn.gpu_addr + offset + AMDGPU_VCN_STACK_SIZE));
 328         WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH,
 329                      upper_32_bits(adev->vcn.gpu_addr + offset + AMDGPU_VCN_STACK_SIZE));
 330         WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_OFFSET2, 0);
 331         WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_SIZE2, AMDGPU_VCN_CONTEXT_SIZE);
 332
 333         WREG32_SOC15(UVD, 0, mmUVD_UDEC_ADDR_CONFIG,
 334                         adev->gfx.config.gb_addr_config);
 335         WREG32_SOC15(UVD, 0, mmUVD_UDEC_DB_ADDR_CONFIG,
 336                         adev->gfx.config.gb_addr_config);
 337         WREG32_SOC15(UVD, 0, mmUVD_UDEC_DBW_ADDR_CONFIG,
 338                         adev->gfx.config.gb_addr_config);
 339         WREG32_SOC15(UVD, 0, mmUVD_UDEC_DBW_UV_ADDR_CONFIG,
 340                         adev->gfx.config.gb_addr_config);
 341         WREG32_SOC15(UVD, 0, mmUVD_MIF_CURR_ADDR_CONFIG,
 342                         adev->gfx.config.gb_addr_config);
 343         WREG32_SOC15(UVD, 0, mmUVD_MIF_CURR_UV_ADDR_CONFIG,
 344                         adev->gfx.config.gb_addr_config);
 345         WREG32_SOC15(UVD, 0, mmUVD_MIF_RECON1_ADDR_CONFIG,
 346                         adev->gfx.config.gb_addr_config);
 347         WREG32_SOC15(UVD, 0, mmUVD_MIF_RECON1_UV_ADDR_CONFIG,
 348                         adev->gfx.config.gb_addr_config);
 349         WREG32_SOC15(UVD, 0, mmUVD_MIF_REF_ADDR_CONFIG,
 350                         adev->gfx.config.gb_addr_config);
 351         WREG32_SOC15(UVD, 0, mmUVD_MIF_REF_UV_ADDR_CONFIG,
 352                         adev->gfx.config.gb_addr_config);
 353         WREG32_SOC15(UVD, 0, mmUVD_JPEG_ADDR_CONFIG,
 354                         adev->gfx.config.gb_addr_config);
 355         WREG32_SOC15(UVD, 0, mmUVD_JPEG_UV_ADDR_CONFIG,
 356                         adev->gfx.config.gb_addr_config);
 357 }
 358
 359 static void vcn_v1_0_mc_resume_dpg_mode(struct amdgpu_device *adev)
 360 {
 361         uint32_t size = AMDGPU_GPU_PAGE_ALIGN(adev->vcn.fw->size + 4);
 362         uint32_t offset;
 363
 364         /* cache window 0: fw */
 365         if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
 366                 WREG32_SOC15_DPG_MODE(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW,
 367                              (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN].tmr_mc_addr_lo),
 368                              0xFFFFFFFF, 0);
 369                 WREG32_SOC15_DPG_MODE(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH,
 370                              (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN].tmr_mc_addr_hi),
 371                              0xFFFFFFFF, 0);
 372                 WREG32_SOC15_DPG_MODE(UVD, 0, mmUVD_VCPU_CACHE_OFFSET0, 0,
 373                              0xFFFFFFFF, 0);
 374                 offset = 0;
 375         } else {
 376                 WREG32_SOC15_DPG_MODE(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW,
 377                         lower_32_bits(adev->vcn.gpu_addr), 0xFFFFFFFF, 0);
 378                 WREG32_SOC15_DPG_MODE(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH,
 379                         upper_32_bits(adev->vcn.gpu_addr), 0xFFFFFFFF, 0);
 380                 offset = size;
 381                 WREG32_SOC15_DPG_MODE(UVD, 0, mmUVD_VCPU_CACHE_OFFSET0,
 382                              AMDGPU_UVD_FIRMWARE_OFFSET >> 3, 0xFFFFFFFF, 0);
 383         }
 384
 385         WREG32_SOC15_DPG_MODE(UVD, 0, mmUVD_VCPU_CACHE_SIZE0, size, 0xFFFFFFFF, 0);
 386
 387         /* cache window 1: stack */
 388         WREG32_SOC15_DPG_MODE(UVD, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW,
 389                      lower_32_bits(adev->vcn.gpu_addr + offset), 0xFFFFFFFF, 0);
 390         WREG32_SOC15_DPG_MODE(UVD, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH,
 391                      upper_32_bits(adev->vcn.gpu_addr + offset), 0xFFFFFFFF, 0);
 392         WREG32_SOC15_DPG_MODE(UVD, 0, mmUVD_VCPU_CACHE_OFFSET1, 0,
 393                              0xFFFFFFFF, 0);
 394         WREG32_SOC15_DPG_MODE(UVD, 0, mmUVD_VCPU_CACHE_SIZE1, AMDGPU_VCN_STACK_SIZE,
 395                              0xFFFFFFFF, 0);
 396
 397         /* cache window 2: context */
 398         WREG32_SOC15_DPG_MODE(UVD, 0, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW,
 399                      lower_32_bits(adev->vcn.gpu_addr + offset + AMDGPU_VCN_STACK_SIZE),
 400                              0xFFFFFFFF, 0);
 401         WREG32_SOC15_DPG_MODE(UVD, 0, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH,
 402                      upper_32_bits(adev->vcn.gpu_addr + offset + AMDGPU_VCN_STACK_SIZE),
 403                              0xFFFFFFFF, 0);
 404         WREG32_SOC15_DPG_MODE(UVD, 0, mmUVD_VCPU_CACHE_OFFSET2, 0, 0xFFFFFFFF, 0);
 405         WREG32_SOC15_DPG_MODE(UVD, 0, mmUVD_VCPU_CACHE_SIZE2, AMDGPU_VCN_CONTEXT_SIZE,
 406                              0xFFFFFFFF, 0);
 407
 408         /* VCN global tiling registers */
 409         WREG32_SOC15_DPG_MODE(UVD, 0, mmUVD_UDEC_ADDR_CONFIG,
 410                         adev->gfx.config.gb_addr_config, 0xFFFFFFFF, 0);
 411         WREG32_SOC15_DPG_MODE(UVD, 0, mmUVD_UDEC_DB_ADDR_CONFIG,
 412                         adev->gfx.config.gb_addr_config, 0xFFFFFFFF, 0);
 413         WREG32_SOC15_DPG_MODE(UVD, 0, mmUVD_UDEC_DBW_ADDR_CONFIG,
 414                         adev->gfx.config.gb_addr_config, 0xFFFFFFFF, 0);
 415         WREG32_SOC15_DPG_MODE(UVD, 0, mmUVD_UDEC_DBW_UV_ADDR_CONFIG,
 416                 adev->gfx.config.gb_addr_config, 0xFFFFFFFF, 0);
 417         WREG32_SOC15_DPG_MODE(UVD, 0, mmUVD_MIF_CURR_ADDR_CONFIG,
 418                 adev->gfx.config.gb_addr_config, 0xFFFFFFFF, 0);
 419         WREG32_SOC15_DPG_MODE(UVD, 0, mmUVD_MIF_CURR_UV_ADDR_CONFIG,
 420                 adev->gfx.config.gb_addr_config, 0xFFFFFFFF, 0);
 421         WREG32_SOC15_DPG_MODE(UVD, 0, mmUVD_MIF_RECON1_ADDR_CONFIG,
 422                 adev->gfx.config.gb_addr_config, 0xFFFFFFFF, 0);
 423         WREG32_SOC15_DPG_MODE(UVD, 0, mmUVD_MIF_RECON1_UV_ADDR_CONFIG,
 424                 adev->gfx.config.gb_addr_config, 0xFFFFFFFF, 0);
 425         WREG32_SOC15_DPG_MODE(UVD, 0, mmUVD_MIF_REF_ADDR_CONFIG,
 426                 adev->gfx.config.gb_addr_config, 0xFFFFFFFF, 0);
 427         WREG32_SOC15_DPG_MODE(UVD, 0, mmUVD_MIF_REF_UV_ADDR_CONFIG,
 428                 adev->gfx.config.gb_addr_config, 0xFFFFFFFF, 0);
 429 }
 430
 431 /**
 432  * vcn_v1_0_disable_clock_gating - disable VCN clock gating
 433  *
 434  * @adev: amdgpu_device pointer
 435  * @sw: enable SW clock gating
 436  *
 437  * Disable clock gating for VCN block
 438  */
 439 static void vcn_v1_0_disable_clock_gating(struct amdgpu_device *adev)
 440 {
 441         uint32_t data;
 442
 443         /* JPEG disable CGC */
 444         data = RREG32_SOC15(VCN, 0, mmJPEG_CGC_CTRL);
 445
 446         if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)
 447                 data |= 1 << JPEG_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
 448         else
 449                 data &= ~JPEG_CGC_CTRL__DYN_CLOCK_MODE_MASK;
 450
 451         data |= 1 << JPEG_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
 452         data |= 4 << JPEG_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
 453         WREG32_SOC15(VCN, 0, mmJPEG_CGC_CTRL, data);
 454
 455         data = RREG32_SOC15(VCN, 0, mmJPEG_CGC_GATE);
 456         data &= ~(JPEG_CGC_GATE__JPEG_MASK | JPEG_CGC_GATE__JPEG2_MASK);
 457         WREG32_SOC15(VCN, 0, mmJPEG_CGC_GATE, data);
 458
 459         /* UVD disable CGC */
 460         data = RREG32_SOC15(VCN, 0, mmUVD_CGC_CTRL);
 461         if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)
 462                 data |= 1 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
 463         else
 464                 data &= ~ UVD_CGC_CTRL__DYN_CLOCK_MODE_MASK;
 465
 466         data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
 467         data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
 468         WREG32_SOC15(VCN, 0, mmUVD_CGC_CTRL, data);
 469
 470         data = RREG32_SOC15(VCN, 0, mmUVD_CGC_GATE);
 471         data &= ~(UVD_CGC_GATE__SYS_MASK
 472                 | UVD_CGC_GATE__UDEC_MASK
 473                 | UVD_CGC_GATE__MPEG2_MASK
 474                 | UVD_CGC_GATE__REGS_MASK
 475                 | UVD_CGC_GATE__RBC_MASK
 476                 | UVD_CGC_GATE__LMI_MC_MASK
 477                 | UVD_CGC_GATE__LMI_UMC_MASK
 478                 | UVD_CGC_GATE__IDCT_MASK
 479                 | UVD_CGC_GATE__MPRD_MASK
 480                 | UVD_CGC_GATE__MPC_MASK
 481                 | UVD_CGC_GATE__LBSI_MASK
 482                 | UVD_CGC_GATE__LRBBM_MASK
 483                 | UVD_CGC_GATE__UDEC_RE_MASK
 484                 | UVD_CGC_GATE__UDEC_CM_MASK
 485                 | UVD_CGC_GATE__UDEC_IT_MASK
 486                 | UVD_CGC_GATE__UDEC_DB_MASK
 487                 | UVD_CGC_GATE__UDEC_MP_MASK
 488                 | UVD_CGC_GATE__WCB_MASK
 489                 | UVD_CGC_GATE__VCPU_MASK
 490                 | UVD_CGC_GATE__SCPU_MASK);
 491         WREG32_SOC15(VCN, 0, mmUVD_CGC_GATE, data);
 492
 493         data = RREG32_SOC15(VCN, 0, mmUVD_CGC_CTRL);
 494         data &= ~(UVD_CGC_CTRL__UDEC_RE_MODE_MASK
 495                 | UVD_CGC_CTRL__UDEC_CM_MODE_MASK
 496                 | UVD_CGC_CTRL__UDEC_IT_MODE_MASK
 497                 | UVD_CGC_CTRL__UDEC_DB_MODE_MASK
 498                 | UVD_CGC_CTRL__UDEC_MP_MODE_MASK
 499                 | UVD_CGC_CTRL__SYS_MODE_MASK
 500                 | UVD_CGC_CTRL__UDEC_MODE_MASK
 501                 | UVD_CGC_CTRL__MPEG2_MODE_MASK
 502                 | UVD_CGC_CTRL__REGS_MODE_MASK
 503                 | UVD_CGC_CTRL__RBC_MODE_MASK
 504                 | UVD_CGC_CTRL__LMI_MC_MODE_MASK
 505                 | UVD_CGC_CTRL__LMI_UMC_MODE_MASK
 506                 | UVD_CGC_CTRL__IDCT_MODE_MASK
 507                 | UVD_CGC_CTRL__MPRD_MODE_MASK
 508                 | UVD_CGC_CTRL__MPC_MODE_MASK
 509                 | UVD_CGC_CTRL__LBSI_MODE_MASK
 510                 | UVD_CGC_CTRL__LRBBM_MODE_MASK
 511                 | UVD_CGC_CTRL__WCB_MODE_MASK
 512                 | UVD_CGC_CTRL__VCPU_MODE_MASK
 513                 | UVD_CGC_CTRL__SCPU_MODE_MASK);
 514         WREG32_SOC15(VCN, 0, mmUVD_CGC_CTRL, data);
 515
 516         /* turn on */
 517         data = RREG32_SOC15(VCN, 0, mmUVD_SUVD_CGC_GATE);
 518         data |= (UVD_SUVD_CGC_GATE__SRE_MASK
 519                 | UVD_SUVD_CGC_GATE__SIT_MASK
 520                 | UVD_SUVD_CGC_GATE__SMP_MASK
 521                 | UVD_SUVD_CGC_GATE__SCM_MASK
 522                 | UVD_SUVD_CGC_GATE__SDB_MASK
 523                 | UVD_SUVD_CGC_GATE__SRE_H264_MASK
 524                 | UVD_SUVD_CGC_GATE__SRE_HEVC_MASK
 525                 | UVD_SUVD_CGC_GATE__SIT_H264_MASK
 526                 | UVD_SUVD_CGC_GATE__SIT_HEVC_MASK
 527                 | UVD_SUVD_CGC_GATE__SCM_H264_MASK
 528                 | UVD_SUVD_CGC_GATE__SCM_HEVC_MASK
 529                 | UVD_SUVD_CGC_GATE__SDB_H264_MASK
 530                 | UVD_SUVD_CGC_GATE__SDB_HEVC_MASK
 531                 | UVD_SUVD_CGC_GATE__SCLR_MASK
 532                 | UVD_SUVD_CGC_GATE__UVD_SC_MASK
 533                 | UVD_SUVD_CGC_GATE__ENT_MASK
 534                 | UVD_SUVD_CGC_GATE__SIT_HEVC_DEC_MASK
 535                 | UVD_SUVD_CGC_GATE__SIT_HEVC_ENC_MASK
 536                 | UVD_SUVD_CGC_GATE__SITE_MASK
 537                 | UVD_SUVD_CGC_GATE__SRE_VP9_MASK
 538                 | UVD_SUVD_CGC_GATE__SCM_VP9_MASK
 539                 | UVD_SUVD_CGC_GATE__SIT_VP9_DEC_MASK
 540                 | UVD_SUVD_CGC_GATE__SDB_VP9_MASK
 541                 | UVD_SUVD_CGC_GATE__IME_HEVC_MASK);
 542         WREG32_SOC15(VCN, 0, mmUVD_SUVD_CGC_GATE, data);
 543
 544         data = RREG32_SOC15(VCN, 0, mmUVD_SUVD_CGC_CTRL);
 545         data &= ~(UVD_SUVD_CGC_CTRL__SRE_MODE_MASK
 546                 | UVD_SUVD_CGC_CTRL__SIT_MODE_MASK
 547                 | UVD_SUVD_CGC_CTRL__SMP_MODE_MASK
 548                 | UVD_SUVD_CGC_CTRL__SCM_MODE_MASK
 549                 | UVD_SUVD_CGC_CTRL__SDB_MODE_MASK
 550                 | UVD_SUVD_CGC_CTRL__SCLR_MODE_MASK
 551                 | UVD_SUVD_CGC_CTRL__UVD_SC_MODE_MASK
 552                 | UVD_SUVD_CGC_CTRL__ENT_MODE_MASK
 553                 | UVD_SUVD_CGC_CTRL__IME_MODE_MASK
 554                 | UVD_SUVD_CGC_CTRL__SITE_MODE_MASK);
 555         WREG32_SOC15(VCN, 0, mmUVD_SUVD_CGC_CTRL, data);
 556 }
 557
 558 /**
 559  * vcn_v1_0_enable_clock_gating - enable VCN clock gating
 560  *
 561  * @adev: amdgpu_device pointer
 562  * @sw: enable SW clock gating
 563  *
 564  * Enable clock gating for VCN block
 565  */
 566 static void vcn_v1_0_enable_clock_gating(struct amdgpu_device *adev)
 567 {
 568         uint32_t data = 0;
 569
 570         /* enable JPEG CGC */
 571         data = RREG32_SOC15(VCN, 0, mmJPEG_CGC_CTRL);
 572         if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)
 573                 data |= 1 << JPEG_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
 574         else
 575                 data |= 0 << JPEG_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
 576         data |= 1 << JPEG_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
 577         data |= 4 << JPEG_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
 578         WREG32_SOC15(VCN, 0, mmJPEG_CGC_CTRL, data);
 579
 580         data = RREG32_SOC15(VCN, 0, mmJPEG_CGC_GATE);
 581         data |= (JPEG_CGC_GATE__JPEG_MASK | JPEG_CGC_GATE__JPEG2_MASK);
 582         WREG32_SOC15(VCN, 0, mmJPEG_CGC_GATE, data);
 583
 584         /* enable UVD CGC */
 585         data = RREG32_SOC15(VCN, 0, mmUVD_CGC_CTRL);
 586         if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)
 587                 data |= 1 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
 588         else
 589                 data |= 0 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
 590         data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
 591         data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
 592         WREG32_SOC15(VCN, 0, mmUVD_CGC_CTRL, data);
 593
 594         data = RREG32_SOC15(VCN, 0, mmUVD_CGC_CTRL);
 595         data |= (UVD_CGC_CTRL__UDEC_RE_MODE_MASK
 596                 | UVD_CGC_CTRL__UDEC_CM_MODE_MASK
 597                 | UVD_CGC_CTRL__UDEC_IT_MODE_MASK
 598                 | UVD_CGC_CTRL__UDEC_DB_MODE_MASK
 599                 | UVD_CGC_CTRL__UDEC_MP_MODE_MASK
 600                 | UVD_CGC_CTRL__SYS_MODE_MASK
 601                 | UVD_CGC_CTRL__UDEC_MODE_MASK
 602                 | UVD_CGC_CTRL__MPEG2_MODE_MASK
 603                 | UVD_CGC_CTRL__REGS_MODE_MASK
 604                 | UVD_CGC_CTRL__RBC_MODE_MASK
 605                 | UVD_CGC_CTRL__LMI_MC_MODE_MASK
 606                 | UVD_CGC_CTRL__LMI_UMC_MODE_MASK
 607                 | UVD_CGC_CTRL__IDCT_MODE_MASK
 608                 | UVD_CGC_CTRL__MPRD_MODE_MASK
 609                 | UVD_CGC_CTRL__MPC_MODE_MASK
 610                 | UVD_CGC_CTRL__LBSI_MODE_MASK
 611                 | UVD_CGC_CTRL__LRBBM_MODE_MASK
 612                 | UVD_CGC_CTRL__WCB_MODE_MASK
 613                 | UVD_CGC_CTRL__VCPU_MODE_MASK
 614                 | UVD_CGC_CTRL__SCPU_MODE_MASK);
 615         WREG32_SOC15(VCN, 0, mmUVD_CGC_CTRL, data);
 616
 617         data = RREG32_SOC15(VCN, 0, mmUVD_SUVD_CGC_CTRL);
 618         data |= (UVD_SUVD_CGC_CTRL__SRE_MODE_MASK
 619                 | UVD_SUVD_CGC_CTRL__SIT_MODE_MASK
 620                 | UVD_SUVD_CGC_CTRL__SMP_MODE_MASK
 621                 | UVD_SUVD_CGC_CTRL__SCM_MODE_MASK
 622                 | UVD_SUVD_CGC_CTRL__SDB_MODE_MASK
 623                 | UVD_SUVD_CGC_CTRL__SCLR_MODE_MASK
 624                 | UVD_SUVD_CGC_CTRL__UVD_SC_MODE_MASK
 625                 | UVD_SUVD_CGC_CTRL__ENT_MODE_MASK
 626                 | UVD_SUVD_CGC_CTRL__IME_MODE_MASK
 627                 | UVD_SUVD_CGC_CTRL__SITE_MODE_MASK);
 628         WREG32_SOC15(VCN, 0, mmUVD_SUVD_CGC_CTRL, data);
 629 }
 630
 631 static void vcn_v1_0_clock_gating_dpg_mode(struct amdgpu_device *adev, uint8_t sram_sel)
 632 {
 633         uint32_t reg_data = 0;
 634
 635         /* disable JPEG CGC */
 636         if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)
 637                 reg_data = 1 << JPEG_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
 638         else
 639                 reg_data = 0 << JPEG_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
 640         reg_data |= 1 << JPEG_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
 641         reg_data |= 4 << JPEG_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
 642         WREG32_SOC15_DPG_MODE(UVD, 0, mmJPEG_CGC_CTRL, reg_data, 0xFFFFFFFF, sram_sel);
 643
 644         WREG32_SOC15_DPG_MODE(UVD, 0, mmJPEG_CGC_GATE, 0, 0xFFFFFFFF, sram_sel);
 645
 646         /* enable sw clock gating control */
 647         if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)
 648                 reg_data = 1 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
 649         else
 650                 reg_data = 0 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
 651         reg_data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
 652         reg_data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
 653         reg_data &= ~(UVD_CGC_CTRL__UDEC_RE_MODE_MASK |
 654                  UVD_CGC_CTRL__UDEC_CM_MODE_MASK |
 655                  UVD_CGC_CTRL__UDEC_IT_MODE_MASK |
 656                  UVD_CGC_CTRL__UDEC_DB_MODE_MASK |
 657                  UVD_CGC_CTRL__UDEC_MP_MODE_MASK |
 658                  UVD_CGC_CTRL__SYS_MODE_MASK |
 659                  UVD_CGC_CTRL__UDEC_MODE_MASK |
 660                  UVD_CGC_CTRL__MPEG2_MODE_MASK |
 661                  UVD_CGC_CTRL__REGS_MODE_MASK |
 662                  UVD_CGC_CTRL__RBC_MODE_MASK |
 663                  UVD_CGC_CTRL__LMI_MC_MODE_MASK |
 664                  UVD_CGC_CTRL__LMI_UMC_MODE_MASK |
 665                  UVD_CGC_CTRL__IDCT_MODE_MASK |
 666                  UVD_CGC_CTRL__MPRD_MODE_MASK |
 667                  UVD_CGC_CTRL__MPC_MODE_MASK |
 668                  UVD_CGC_CTRL__LBSI_MODE_MASK |
 669                  UVD_CGC_CTRL__LRBBM_MODE_MASK |
 670                  UVD_CGC_CTRL__WCB_MODE_MASK |
 671                  UVD_CGC_CTRL__VCPU_MODE_MASK |
 672                  UVD_CGC_CTRL__SCPU_MODE_MASK);
 673         WREG32_SOC15_DPG_MODE(UVD, 0, mmUVD_CGC_CTRL, reg_data, 0xFFFFFFFF, sram_sel);
 674
 675         /* turn off clock gating */
 676         WREG32_SOC15_DPG_MODE(UVD, 0, mmUVD_CGC_GATE, 0, 0xFFFFFFFF, sram_sel);
 677
 678         /* turn on SUVD clock gating */
 679         WREG32_SOC15_DPG_MODE(UVD, 0, mmUVD_SUVD_CGC_GATE, 1, 0xFFFFFFFF, sram_sel);
 680
 681         /* turn on sw mode in UVD_SUVD_CGC_CTRL */
 682         WREG32_SOC15_DPG_MODE(UVD, 0, mmUVD_SUVD_CGC_CTRL, 0, 0xFFFFFFFF, sram_sel);
 683 }
 684
 685 static void vcn_1_0_disable_static_power_gating(struct amdgpu_device *adev)
 686 {
 687         uint32_t data = 0;
 688         int ret;
 689
 690         if (adev->pg_flags & AMD_PG_SUPPORT_VCN) {
 691                 data = (1 << UVD_PGFSM_CONFIG__UVDM_PWR_CONFIG__SHIFT
 692                         | 1 << UVD_PGFSM_CONFIG__UVDU_PWR_CONFIG__SHIFT
 693                         | 2 << UVD_PGFSM_CONFIG__UVDF_PWR_CONFIG__SHIFT
 694                         | 2 << UVD_PGFSM_CONFIG__UVDC_PWR_CONFIG__SHIFT
 695                         | 2 << UVD_PGFSM_CONFIG__UVDB_PWR_CONFIG__SHIFT
 696                         | 2 << UVD_PGFSM_CONFIG__UVDIL_PWR_CONFIG__SHIFT
 697                         | 2 << UVD_PGFSM_CONFIG__UVDIR_PWR_CONFIG__SHIFT
 698                         | 2 << UVD_PGFSM_CONFIG__UVDTD_PWR_CONFIG__SHIFT
 699                         | 2 << UVD_PGFSM_CONFIG__UVDTE_PWR_CONFIG__SHIFT
 700                         | 2 << UVD_PGFSM_CONFIG__UVDE_PWR_CONFIG__SHIFT
 701                         | 2 << UVD_PGFSM_CONFIG__UVDW_PWR_CONFIG__SHIFT);
 702
 703                 WREG32_SOC15(VCN, 0, mmUVD_PGFSM_CONFIG, data);
 704                 SOC15_WAIT_ON_RREG(VCN, 0, mmUVD_PGFSM_STATUS, UVD_PGFSM_STATUS__UVDM_UVDU_PWR_ON, 0xFFFFFF, ret);
 705         } else {
 706                 data = (1 << UVD_PGFSM_CONFIG__UVDM_PWR_CONFIG__SHIFT
 707                         | 1 << UVD_PGFSM_CONFIG__UVDU_PWR_CONFIG__SHIFT
 708                         | 1 << UVD_PGFSM_CONFIG__UVDF_PWR_CONFIG__SHIFT
 709                         | 1 << UVD_PGFSM_CONFIG__UVDC_PWR_CONFIG__SHIFT
 710                         | 1 << UVD_PGFSM_CONFIG__UVDB_PWR_CONFIG__SHIFT
 711                         | 1 << UVD_PGFSM_CONFIG__UVDIL_PWR_CONFIG__SHIFT
 712                         | 1 << UVD_PGFSM_CONFIG__UVDIR_PWR_CONFIG__SHIFT
 713                         | 1 << UVD_PGFSM_CONFIG__UVDTD_PWR_CONFIG__SHIFT
 714                         | 1 << UVD_PGFSM_CONFIG__UVDTE_PWR_CONFIG__SHIFT
 715                         | 1 << UVD_PGFSM_CONFIG__UVDE_PWR_CONFIG__SHIFT
 716                         | 1 << UVD_PGFSM_CONFIG__UVDW_PWR_CONFIG__SHIFT);
 717                 WREG32_SOC15(VCN, 0, mmUVD_PGFSM_CONFIG, data);
 718                 SOC15_WAIT_ON_RREG(VCN, 0, mmUVD_PGFSM_STATUS, 0,  0xFFFFFFFF, ret);
 719         }
 720
 721         /* polling UVD_PGFSM_STATUS to confirm UVDM_PWR_STATUS , UVDU_PWR_STATUS are 0 (power on) */
 722
 723         data = RREG32_SOC15(VCN, 0, mmUVD_POWER_STATUS);
 724         data &= ~0x103;
 725         if (adev->pg_flags & AMD_PG_SUPPORT_VCN)
 726                 data |= UVD_PGFSM_CONFIG__UVDM_UVDU_PWR_ON | UVD_POWER_STATUS__UVD_PG_EN_MASK;
 727
 728         WREG32_SOC15(VCN, 0, mmUVD_POWER_STATUS, data);
 729 }
 730
 731 static void vcn_1_0_enable_static_power_gating(struct amdgpu_device *adev)
 732 {
 733         uint32_t data = 0;
 734         int ret;
 735
 736         if (adev->pg_flags & AMD_PG_SUPPORT_VCN) {
 737                 /* Before power off, this indicator has to be turned on */
 738                 data = RREG32_SOC15(VCN, 0, mmUVD_POWER_STATUS);
 739                 data &= ~UVD_POWER_STATUS__UVD_POWER_STATUS_MASK;
 740                 data |= UVD_POWER_STATUS__UVD_POWER_STATUS_TILES_OFF;
 741                 WREG32_SOC15(VCN, 0, mmUVD_POWER_STATUS, data);
 742
 743
 744                 data = (2 << UVD_PGFSM_CONFIG__UVDM_PWR_CONFIG__SHIFT
 745                         | 2 << UVD_PGFSM_CONFIG__UVDU_PWR_CONFIG__SHIFT
 746                         | 2 << UVD_PGFSM_CONFIG__UVDF_PWR_CONFIG__SHIFT
 747                         | 2 << UVD_PGFSM_CONFIG__UVDC_PWR_CONFIG__SHIFT
 748                         | 2 << UVD_PGFSM_CONFIG__UVDB_PWR_CONFIG__SHIFT
 749                         | 2 << UVD_PGFSM_CONFIG__UVDIL_PWR_CONFIG__SHIFT
 750                         | 2 << UVD_PGFSM_CONFIG__UVDIR_PWR_CONFIG__SHIFT
 751                         | 2 << UVD_PGFSM_CONFIG__UVDTD_PWR_CONFIG__SHIFT
 752                         | 2 << UVD_PGFSM_CONFIG__UVDTE_PWR_CONFIG__SHIFT
 753                         | 2 << UVD_PGFSM_CONFIG__UVDE_PWR_CONFIG__SHIFT
 754                         | 2 << UVD_PGFSM_CONFIG__UVDW_PWR_CONFIG__SHIFT);
 755
 756                 WREG32_SOC15(VCN, 0, mmUVD_PGFSM_CONFIG, data);
 757
 758                 data = (2 << UVD_PGFSM_STATUS__UVDM_PWR_STATUS__SHIFT
 759                         | 2 << UVD_PGFSM_STATUS__UVDU_PWR_STATUS__SHIFT
 760                         | 2 << UVD_PGFSM_STATUS__UVDF_PWR_STATUS__SHIFT
 761                         | 2 << UVD_PGFSM_STATUS__UVDC_PWR_STATUS__SHIFT
 762                         | 2 << UVD_PGFSM_STATUS__UVDB_PWR_STATUS__SHIFT
 763                         | 2 << UVD_PGFSM_STATUS__UVDIL_PWR_STATUS__SHIFT
 764                         | 2 << UVD_PGFSM_STATUS__UVDIR_PWR_STATUS__SHIFT
 765                         | 2 << UVD_PGFSM_STATUS__UVDTD_PWR_STATUS__SHIFT
 766                         | 2 << UVD_PGFSM_STATUS__UVDTE_PWR_STATUS__SHIFT
 767                         | 2 << UVD_PGFSM_STATUS__UVDE_PWR_STATUS__SHIFT
 768                         | 2 << UVD_PGFSM_STATUS__UVDW_PWR_STATUS__SHIFT);
 769                 SOC15_WAIT_ON_RREG(VCN, 0, mmUVD_PGFSM_STATUS, data, 0xFFFFFFFF, ret);
 770         }
 771 }
 772
 773 /**
 774  * vcn_v1_0_start - start VCN block
 775  *
 776  * @adev: amdgpu_device pointer
 777  *
 778  * Setup and start the VCN block
 779  */
 780 static int vcn_v1_0_start_spg_mode(struct amdgpu_device *adev)
 781 {
 782         struct amdgpu_ring *ring = &adev->vcn.ring_dec;
 783         uint32_t rb_bufsz, tmp;
 784         uint32_t lmi_swap_cntl;
 785         int i, j, r;
 786
 787         /* disable byte swapping */
 788         lmi_swap_cntl = 0;
 789
 790         vcn_1_0_disable_static_power_gating(adev);
 791
 792         tmp = RREG32_SOC15(UVD, 0, mmUVD_STATUS) | UVD_STATUS__UVD_BUSY;
 793         WREG32_SOC15(UVD, 0, mmUVD_STATUS, tmp);
 794
 795         /* disable clock gating */
 796         vcn_v1_0_disable_clock_gating(adev);
 797
 798         /* disable interupt */
 799         WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_MASTINT_EN), 0,
 800                         ~UVD_MASTINT_EN__VCPU_EN_MASK);
 801
 802         /* initialize VCN memory controller */
 803         tmp = RREG32_SOC15(UVD, 0, mmUVD_LMI_CTRL);
 804         WREG32_SOC15(UVD, 0, mmUVD_LMI_CTRL, tmp                |
 805                 UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK |
 806                 UVD_LMI_CTRL__MASK_MC_URGENT_MASK                       |
 807                 UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK            |
 808                 UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK);
 809
 810 #ifdef __BIG_ENDIAN
 811         /* swap (8 in 32) RB and IB */
 812         lmi_swap_cntl = 0xa;
 813 #endif
 814         WREG32_SOC15(UVD, 0, mmUVD_LMI_SWAP_CNTL, lmi_swap_cntl);
 815
 816         tmp = RREG32_SOC15(UVD, 0, mmUVD_MPC_CNTL);
 817         tmp &= ~UVD_MPC_CNTL__REPLACEMENT_MODE_MASK;
 818         tmp |= 0x2 << UVD_MPC_CNTL__REPLACEMENT_MODE__SHIFT;
 819         WREG32_SOC15(UVD, 0, mmUVD_MPC_CNTL, tmp);
 820
 821         WREG32_SOC15(UVD, 0, mmUVD_MPC_SET_MUXA0,
 822                 ((0x1 << UVD_MPC_SET_MUXA0__VARA_1__SHIFT) |
 823                 (0x2 << UVD_MPC_SET_MUXA0__VARA_2__SHIFT) |
 824                 (0x3 << UVD_MPC_SET_MUXA0__VARA_3__SHIFT) |
 825                 (0x4 << UVD_MPC_SET_MUXA0__VARA_4__SHIFT)));
 826
 827         WREG32_SOC15(UVD, 0, mmUVD_MPC_SET_MUXB0,
 828                 ((0x1 << UVD_MPC_SET_MUXB0__VARB_1__SHIFT) |
 829                 (0x2 << UVD_MPC_SET_MUXB0__VARB_2__SHIFT) |
 830                 (0x3 << UVD_MPC_SET_MUXB0__VARB_3__SHIFT) |
 831                 (0x4 << UVD_MPC_SET_MUXB0__VARB_4__SHIFT)));
 832
 833         WREG32_SOC15(UVD, 0, mmUVD_MPC_SET_MUX,
 834                 ((0x0 << UVD_MPC_SET_MUX__SET_0__SHIFT) |
 835                 (0x1 << UVD_MPC_SET_MUX__SET_1__SHIFT) |
 836                 (0x2 << UVD_MPC_SET_MUX__SET_2__SHIFT)));
 837
 838         vcn_v1_0_mc_resume_spg_mode(adev);
 839
 840         WREG32_SOC15(UVD, 0, mmUVD_REG_XX_MASK, 0x10);
 841         WREG32_SOC15(UVD, 0, mmUVD_RBC_XX_IB_REG_CHECK,
 842                 RREG32_SOC15(UVD, 0, mmUVD_RBC_XX_IB_REG_CHECK) | 0x3);
 843
 844         /* enable VCPU clock */
 845         WREG32_SOC15(UVD, 0, mmUVD_VCPU_CNTL, UVD_VCPU_CNTL__CLK_EN_MASK);
 846
 847         /* boot up the VCPU */
 848         WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_SOFT_RESET), 0,
 849                         ~UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK);
 850
 851         /* enable UMC */
 852         WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_CTRL2), 0,
 853                         ~UVD_LMI_CTRL2__STALL_ARB_UMC_MASK);
 854
 855         tmp = RREG32_SOC15(UVD, 0, mmUVD_SOFT_RESET);
 856         tmp &= ~UVD_SOFT_RESET__LMI_SOFT_RESET_MASK;
 857         tmp &= ~UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK;
 858         WREG32_SOC15(UVD, 0, mmUVD_SOFT_RESET, tmp);
 859
 860         for (i = 0; i < 10; ++i) {
 861                 uint32_t status;
 862
 863                 for (j = 0; j < 100; ++j) {
 864                         status = RREG32_SOC15(UVD, 0, mmUVD_STATUS);
 865                         if (status & UVD_STATUS__IDLE)
 866                                 break;
 867                         mdelay(10);
 868                 }
 869                 r = 0;
 870                 if (status & UVD_STATUS__IDLE)
 871                         break;
 872
 873                 DRM_ERROR("VCN decode not responding, trying to reset the VCPU!!!\n");
 874                 WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_SOFT_RESET),
 875                                 UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK,
 876                                 ~UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK);
 877                 mdelay(10);
 878                 WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_SOFT_RESET), 0,
 879                                 ~UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK);
 880                 mdelay(10);
 881                 r = -1;
 882         }
 883
 884         if (r) {
 885                 DRM_ERROR("VCN decode not responding, giving up!!!\n");
 886                 return r;
 887         }
 888         /* enable master interrupt */
 889         WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_MASTINT_EN),
 890                 UVD_MASTINT_EN__VCPU_EN_MASK, ~UVD_MASTINT_EN__VCPU_EN_MASK);
 891
 892         /* enable system interrupt for JRBC, TODO: move to set interrupt*/
 893         WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_SYS_INT_EN),
 894                 UVD_SYS_INT_EN__UVD_JRBC_EN_MASK,
 895                 ~UVD_SYS_INT_EN__UVD_JRBC_EN_MASK);
 896
 897         /* clear the busy bit of UVD_STATUS */
 898         tmp = RREG32_SOC15(UVD, 0, mmUVD_STATUS) & ~UVD_STATUS__UVD_BUSY;
 899         WREG32_SOC15(UVD, 0, mmUVD_STATUS, tmp);
 900
 901         /* force RBC into idle state */
 902         rb_bufsz = order_base_2(ring->ring_size);
 903         tmp = REG_SET_FIELD(0, UVD_RBC_RB_CNTL, RB_BUFSZ, rb_bufsz);
 904         tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_BLKSZ, 1);
 905         tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_FETCH, 1);
 906         tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_UPDATE, 1);
 907         tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_RPTR_WR_EN, 1);
 908         WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_CNTL, tmp);
 909
 910         /* set the write pointer delay */
 911         WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_WPTR_CNTL, 0);
 912
 913         /* set the wb address */
 914         WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_RPTR_ADDR,
 915                         (upper_32_bits(ring->gpu_addr) >> 2));
 916
 917         /* programm the RB_BASE for ring buffer */
 918         WREG32_SOC15(UVD, 0, mmUVD_LMI_RBC_RB_64BIT_BAR_LOW,
 919                         lower_32_bits(ring->gpu_addr));
 920         WREG32_SOC15(UVD, 0, mmUVD_LMI_RBC_RB_64BIT_BAR_HIGH,
 921                         upper_32_bits(ring->gpu_addr));
 922
 923         /* Initialize the ring buffer's read and write pointers */
 924         WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_RPTR, 0);
 925
 926         WREG32_SOC15(UVD, 0, mmUVD_SCRATCH2, 0);
 927
 928         ring->wptr = RREG32_SOC15(UVD, 0, mmUVD_RBC_RB_RPTR);
 929         WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_WPTR,
 930                         lower_32_bits(ring->wptr));
 931
 932         WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_RBC_RB_CNTL), 0,
 933                         ~UVD_RBC_RB_CNTL__RB_NO_FETCH_MASK);
 934
 935         ring = &adev->vcn.ring_enc[0];
 936         WREG32_SOC15(UVD, 0, mmUVD_RB_RPTR, lower_32_bits(ring->wptr));
 937         WREG32_SOC15(UVD, 0, mmUVD_RB_WPTR, lower_32_bits(ring->wptr));
 938         WREG32_SOC15(UVD, 0, mmUVD_RB_BASE_LO, ring->gpu_addr);
 939         WREG32_SOC15(UVD, 0, mmUVD_RB_BASE_HI, upper_32_bits(ring->gpu_addr));
 940         WREG32_SOC15(UVD, 0, mmUVD_RB_SIZE, ring->ring_size / 4);
 941
 942         ring = &adev->vcn.ring_enc[1];
 943         WREG32_SOC15(UVD, 0, mmUVD_RB_RPTR2, lower_32_bits(ring->wptr));
 944         WREG32_SOC15(UVD, 0, mmUVD_RB_WPTR2, lower_32_bits(ring->wptr));
 945         WREG32_SOC15(UVD, 0, mmUVD_RB_BASE_LO2, ring->gpu_addr);
 946         WREG32_SOC15(UVD, 0, mmUVD_RB_BASE_HI2, upper_32_bits(ring->gpu_addr));
 947         WREG32_SOC15(UVD, 0, mmUVD_RB_SIZE2, ring->ring_size / 4);
 948
 949         ring = &adev->vcn.ring_jpeg;
 950         WREG32_SOC15(UVD, 0, mmUVD_LMI_JRBC_RB_VMID, 0);
 951         WREG32_SOC15(UVD, 0, mmUVD_JRBC_RB_CNTL, UVD_JRBC_RB_CNTL__RB_NO_FETCH_MASK |
 952                         UVD_JRBC_RB_CNTL__RB_RPTR_WR_EN_MASK);
 953         WREG32_SOC15(UVD, 0, mmUVD_LMI_JRBC_RB_64BIT_BAR_LOW, lower_32_bits(ring->gpu_addr));
 954         WREG32_SOC15(UVD, 0, mmUVD_LMI_JRBC_RB_64BIT_BAR_HIGH, upper_32_bits(ring->gpu_addr));
 955         WREG32_SOC15(UVD, 0, mmUVD_JRBC_RB_RPTR, 0);
 956         WREG32_SOC15(UVD, 0, mmUVD_JRBC_RB_WPTR, 0);
 957         WREG32_SOC15(UVD, 0, mmUVD_JRBC_RB_CNTL, UVD_JRBC_RB_CNTL__RB_RPTR_WR_EN_MASK);
 958
 959         /* initialize wptr */
 960         ring->wptr = RREG32_SOC15(UVD, 0, mmUVD_JRBC_RB_WPTR);
 961
 962         /* copy patch commands to the jpeg ring */
 963         vcn_v1_0_jpeg_ring_set_patch_ring(ring,
 964                 (ring->wptr + ring->max_dw * amdgpu_sched_hw_submission));
 965
 966         return 0;
 967 }
 968
 969 static int vcn_v1_0_start_dpg_mode(struct amdgpu_device *adev)
 970 {
 971         struct amdgpu_ring *ring = &adev->vcn.ring_dec;
 972         uint32_t rb_bufsz, tmp;
 973         uint32_t lmi_swap_cntl;
 974
 975         /* disable byte swapping */
 976         lmi_swap_cntl = 0;
 977
 978         vcn_1_0_enable_static_power_gating(adev);
 979
 980         /* enable dynamic power gating mode */
 981         tmp = RREG32_SOC15(UVD, 0, mmUVD_POWER_STATUS);
 982         tmp |= UVD_POWER_STATUS__UVD_PG_MODE_MASK;
 983         tmp |= UVD_POWER_STATUS__UVD_PG_EN_MASK;
 984         WREG32_SOC15(UVD, 0, mmUVD_POWER_STATUS, tmp);
 985
 986         /* enable clock gating */
 987         vcn_v1_0_clock_gating_dpg_mode(adev, 0);
 988
 989         /* enable VCPU clock */
 990         tmp = (0xFF << UVD_VCPU_CNTL__PRB_TIMEOUT_VAL__SHIFT);
 991         tmp |= UVD_VCPU_CNTL__CLK_EN_MASK;
 992         tmp |= UVD_VCPU_CNTL__MIF_WR_LOW_THRESHOLD_BP_MASK;
 993         WREG32_SOC15_DPG_MODE(UVD, 0, mmUVD_VCPU_CNTL, tmp, 0xFFFFFFFF, 0);
 994
 995         /* disable interupt */
 996         WREG32_SOC15_DPG_MODE(UVD, 0, mmUVD_MASTINT_EN,
 997                         0, UVD_MASTINT_EN__VCPU_EN_MASK, 0);
 998
 999         /* initialize VCN memory controller */
1000         WREG32_SOC15_DPG_MODE(UVD, 0, mmUVD_LMI_CTRL,
1001                 (8 << UVD_LMI_CTRL__WRITE_CLEAN_TIMER__SHIFT) |
1002                 UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK |
1003                 UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK |
1004                 UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK |
1005                 UVD_LMI_CTRL__REQ_MODE_MASK |
1006                 UVD_LMI_CTRL__CRC_RESET_MASK |
1007                 UVD_LMI_CTRL__MASK_MC_URGENT_MASK |
1008                 0x00100000L, 0xFFFFFFFF, 0);
1009
1010 #ifdef __BIG_ENDIAN
1011         /* swap (8 in 32) RB and IB */
1012         lmi_swap_cntl = 0xa;
1013 #endif
1014         WREG32_SOC15_DPG_MODE(UVD, 0, mmUVD_LMI_SWAP_CNTL, lmi_swap_cntl, 0xFFFFFFFF, 0);
1015
1016         WREG32_SOC15_DPG_MODE(UVD, 0, mmUVD_MPC_CNTL,
1017                 0x2 << UVD_MPC_CNTL__REPLACEMENT_MODE__SHIFT, 0xFFFFFFFF, 0);
1018
1019         WREG32_SOC15_DPG_MODE(UVD, 0, mmUVD_MPC_SET_MUXA0,
1020                 ((0x1 << UVD_MPC_SET_MUXA0__VARA_1__SHIFT) |
1021                  (0x2 << UVD_MPC_SET_MUXA0__VARA_2__SHIFT) |
1022                  (0x3 << UVD_MPC_SET_MUXA0__VARA_3__SHIFT) |
1023                  (0x4 << UVD_MPC_SET_MUXA0__VARA_4__SHIFT)), 0xFFFFFFFF, 0);
1024
1025         WREG32_SOC15_DPG_MODE(UVD, 0, mmUVD_MPC_SET_MUXB0,
1026                 ((0x1 << UVD_MPC_SET_MUXB0__VARB_1__SHIFT) |
1027                  (0x2 << UVD_MPC_SET_MUXB0__VARB_2__SHIFT) |
1028                  (0x3 << UVD_MPC_SET_MUXB0__VARB_3__SHIFT) |
1029                  (0x4 << UVD_MPC_SET_MUXB0__VARB_4__SHIFT)), 0xFFFFFFFF, 0);
1030
1031         WREG32_SOC15_DPG_MODE(UVD, 0, mmUVD_MPC_SET_MUX,
1032                 ((0x0 << UVD_MPC_SET_MUX__SET_0__SHIFT) |
1033                  (0x1 << UVD_MPC_SET_MUX__SET_1__SHIFT) |
1034                  (0x2 << UVD_MPC_SET_MUX__SET_2__SHIFT)), 0xFFFFFFFF, 0);
1035
1036         vcn_v1_0_mc_resume_dpg_mode(adev);
1037
1038         WREG32_SOC15_DPG_MODE(UVD, 0, mmUVD_REG_XX_MASK, 0x10, 0xFFFFFFFF, 0);
1039         WREG32_SOC15_DPG_MODE(UVD, 0, mmUVD_RBC_XX_IB_REG_CHECK, 0x3, 0xFFFFFFFF, 0);
1040
1041         /* boot up the VCPU */
1042         WREG32_SOC15_DPG_MODE(UVD, 0, mmUVD_SOFT_RESET, 0, 0xFFFFFFFF, 0);
1043
1044         /* enable UMC */
1045         WREG32_SOC15_DPG_MODE(UVD, 0, mmUVD_LMI_CTRL2,
1046                 0x1F << UVD_LMI_CTRL2__RE_OFLD_MIF_WR_REQ_NUM__SHIFT,
1047                 0xFFFFFFFF, 0);
1048
1049         /* enable master interrupt */
1050         WREG32_SOC15_DPG_MODE(UVD, 0, mmUVD_MASTINT_EN,
1051                         UVD_MASTINT_EN__VCPU_EN_MASK, UVD_MASTINT_EN__VCPU_EN_MASK, 0);
1052
1053         vcn_v1_0_clock_gating_dpg_mode(adev, 1);
1054         /* setup mmUVD_LMI_CTRL */
1055         WREG32_SOC15_DPG_MODE(UVD, 0, mmUVD_LMI_CTRL,
1056                 (8 << UVD_LMI_CTRL__WRITE_CLEAN_TIMER__SHIFT) |
1057                 UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK |
1058                 UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK |
1059                 UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK |
1060                 UVD_LMI_CTRL__REQ_MODE_MASK |
1061                 UVD_LMI_CTRL__CRC_RESET_MASK |
1062                 UVD_LMI_CTRL__MASK_MC_URGENT_MASK |
1063                 0x00100000L, 0xFFFFFFFF, 1);
1064
1065         tmp = adev->gfx.config.gb_addr_config;
1066         /* setup VCN global tiling registers */
1067         WREG32_SOC15_DPG_MODE(UVD, 0, mmUVD_JPEG_ADDR_CONFIG, tmp, 0xFFFFFFFF, 1);
1068         WREG32_SOC15_DPG_MODE(UVD, 0, mmUVD_JPEG_UV_ADDR_CONFIG, tmp, 0xFFFFFFFF, 1);
1069
1070         /* enable System Interrupt for JRBC */
1071         WREG32_SOC15_DPG_MODE(UVD, 0, mmUVD_SYS_INT_EN,
1072                                                                         UVD_SYS_INT_EN__UVD_JRBC_EN_MASK, 0xFFFFFFFF, 1);
1073
1074         /* force RBC into idle state */
1075         rb_bufsz = order_base_2(ring->ring_size);
1076         tmp = REG_SET_FIELD(0, UVD_RBC_RB_CNTL, RB_BUFSZ, rb_bufsz);
1077         tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_BLKSZ, 1);
1078         tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_FETCH, 1);
1079         tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_UPDATE, 1);
1080         tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_RPTR_WR_EN, 1);
1081         WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_CNTL, tmp);
1082
1083         /* set the write pointer delay */
1084         WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_WPTR_CNTL, 0);
1085
1086         /* set the wb address */
1087         WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_RPTR_ADDR,
1088                                                                 (upper_32_bits(ring->gpu_addr) >> 2));
1089
1090         /* programm the RB_BASE for ring buffer */
1091         WREG32_SOC15(UVD, 0, mmUVD_LMI_RBC_RB_64BIT_BAR_LOW,
1092                                                                 lower_32_bits(ring->gpu_addr));
1093         WREG32_SOC15(UVD, 0, mmUVD_LMI_RBC_RB_64BIT_BAR_HIGH,
1094                                                                 upper_32_bits(ring->gpu_addr));
1095
1096         /* Initialize the ring buffer's read and write pointers */
1097         WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_RPTR, 0);
1098
1099         WREG32_SOC15(UVD, 0, mmUVD_SCRATCH2, 0);
1100
1101         ring->wptr = RREG32_SOC15(UVD, 0, mmUVD_RBC_RB_RPTR);
1102         WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_WPTR,
1103                                                                 lower_32_bits(ring->wptr));
1104
1105         WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_RBC_RB_CNTL), 0,
1106                         ~UVD_RBC_RB_CNTL__RB_NO_FETCH_MASK);
1107
1108         /* initialize JPEG wptr */
1109         ring = &adev->vcn.ring_jpeg;
1110         ring->wptr = RREG32_SOC15(UVD, 0, mmUVD_JRBC_RB_WPTR);
1111
1112         /* copy patch commands to the jpeg ring */
1113         vcn_v1_0_jpeg_ring_set_patch_ring(ring,
1114                 (ring->wptr + ring->max_dw * amdgpu_sched_hw_submission));
1115
1116         return 0;
1117 }
1118
1119 static int vcn_v1_0_start(struct amdgpu_device *adev)
1120 {
1121         int r;
1122
1123         if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG)
1124                 r = vcn_v1_0_start_dpg_mode(adev);
1125         else
1126                 r = vcn_v1_0_start_spg_mode(adev);
1127         return r;
1128 }
1129
1130 /**
1131  * vcn_v1_0_stop - stop VCN block
1132  *
1133  * @adev: amdgpu_device pointer
1134  *
1135  * stop the VCN block
1136  */
1137 static int vcn_v1_0_stop_spg_mode(struct amdgpu_device *adev)
1138 {
1139         int ret_code, tmp;
1140
1141         SOC15_WAIT_ON_RREG(UVD, 0, mmUVD_STATUS, UVD_STATUS__IDLE, 0x7, ret_code);
1142
1143         tmp = UVD_LMI_STATUS__VCPU_LMI_WRITE_CLEAN_MASK |
1144                 UVD_LMI_STATUS__READ_CLEAN_MASK |
1145                 UVD_LMI_STATUS__WRITE_CLEAN_MASK |
1146                 UVD_LMI_STATUS__WRITE_CLEAN_RAW_MASK;
1147         SOC15_WAIT_ON_RREG(UVD, 0, mmUVD_LMI_STATUS, tmp, tmp, ret_code);
1148
1149         /* put VCPU into reset */
1150         WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_SOFT_RESET),
1151                 UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK,
1152                 ~UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK);
1153
1154         tmp = UVD_LMI_STATUS__UMC_READ_CLEAN_RAW_MASK |
1155                 UVD_LMI_STATUS__UMC_WRITE_CLEAN_RAW_MASK;
1156         SOC15_WAIT_ON_RREG(UVD, 0, mmUVD_LMI_STATUS, tmp, tmp, ret_code);
1157
1158         /* disable VCPU clock */
1159         WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_VCPU_CNTL), 0,
1160                 ~UVD_VCPU_CNTL__CLK_EN_MASK);
1161
1162         /* reset LMI UMC/LMI */
1163         WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_SOFT_RESET),
1164                 UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK,
1165                 ~UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK);
1166
1167         WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_SOFT_RESET),
1168                 UVD_SOFT_RESET__LMI_SOFT_RESET_MASK,
1169                 ~UVD_SOFT_RESET__LMI_SOFT_RESET_MASK);
1170
1171         WREG32_SOC15(UVD, 0, mmUVD_STATUS, 0);
1172
1173         vcn_v1_0_enable_clock_gating(adev);
1174         vcn_1_0_enable_static_power_gating(adev);
1175         return 0;
1176 }
1177
1178 static int vcn_v1_0_stop_dpg_mode(struct amdgpu_device *adev)
1179 {
1180         int ret_code = 0;
1181         uint32_t tmp;
1182
1183         /* Wait for power status to be UVD_POWER_STATUS__UVD_POWER_STATUS_TILES_OFF */
1184         SOC15_WAIT_ON_RREG(UVD, 0, mmUVD_POWER_STATUS,
1185                         UVD_POWER_STATUS__UVD_POWER_STATUS_TILES_OFF,
1186                         UVD_POWER_STATUS__UVD_POWER_STATUS_MASK, ret_code);
1187
1188         /* wait for read ptr to be equal to write ptr */
1189         tmp = RREG32_SOC15(UVD, 0, mmUVD_RB_WPTR);
1190         SOC15_WAIT_ON_RREG(UVD, 0, mmUVD_RB_RPTR, tmp, 0xFFFFFFFF, ret_code);
1191
1192         tmp = RREG32_SOC15(UVD, 0, mmUVD_RB_WPTR2);
1193         SOC15_WAIT_ON_RREG(UVD, 0, mmUVD_RB_RPTR2, tmp, 0xFFFFFFFF, ret_code);
1194
1195         tmp = RREG32_SOC15(UVD, 0, mmUVD_JRBC_RB_WPTR);
1196         SOC15_WAIT_ON_RREG(UVD, 0, mmUVD_JRBC_RB_RPTR, tmp, 0xFFFFFFFF, ret_code);
1197
1198         tmp = RREG32_SOC15(UVD, 0, mmUVD_RBC_RB_WPTR) & 0x7FFFFFFF;
1199         SOC15_WAIT_ON_RREG(UVD, 0, mmUVD_RBC_RB_RPTR, tmp, 0xFFFFFFFF, ret_code);
1200
1201         SOC15_WAIT_ON_RREG(UVD, 0, mmUVD_POWER_STATUS,
1202                 UVD_POWER_STATUS__UVD_POWER_STATUS_TILES_OFF,
1203                 UVD_POWER_STATUS__UVD_POWER_STATUS_MASK, ret_code);
1204
1205         /* disable dynamic power gating mode */
1206         WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_POWER_STATUS), 0,
1207                         ~UVD_POWER_STATUS__UVD_PG_MODE_MASK);
1208
1209         return 0;
1210 }
1211
1212 static int vcn_v1_0_stop(struct amdgpu_device *adev)
1213 {
1214         int r;
1215
1216         if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG)
1217                 r = vcn_v1_0_stop_dpg_mode(adev);
1218         else
1219                 r = vcn_v1_0_stop_spg_mode(adev);
1220
1221         return r;
1222 }
1223
1224 static int vcn_v1_0_pause_dpg_mode(struct amdgpu_device *adev,
1225                                 struct dpg_pause_state *new_state)
1226 {
1227         int ret_code;
1228         uint32_t reg_data = 0;
1229         uint32_t reg_data2 = 0;
1230         struct amdgpu_ring *ring;
1231
1232         /* pause/unpause if state is changed */
1233         if (adev->vcn.pause_state.fw_based != new_state->fw_based) {
1234                 DRM_DEBUG("dpg pause state changed %d:%d -> %d:%d",
1235                         adev->vcn.pause_state.fw_based, adev->vcn.pause_state.jpeg,
1236                         new_state->fw_based, new_state->jpeg);
1237
1238                 reg_data = RREG32_SOC15(UVD, 0, mmUVD_DPG_PAUSE) &
1239                         (~UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK);
1240
1241                 if (new_state->fw_based == VCN_DPG_STATE__PAUSE) {
1242                         ret_code = 0;
1243
1244                         if (!(reg_data & UVD_DPG_PAUSE__JPEG_PAUSE_DPG_ACK_MASK))
1245                                 SOC15_WAIT_ON_RREG(UVD, 0, mmUVD_POWER_STATUS,
1246                                                    UVD_POWER_STATUS__UVD_POWER_STATUS_TILES_OFF,
1247                                                    UVD_POWER_STATUS__UVD_POWER_STATUS_MASK, ret_code);
1248
1249                         if (!ret_code) {
1250                                 /* pause DPG non-jpeg */
1251                                 reg_data |= UVD_DPG_PAUSE__NJ_PAUSE_DPG_REQ_MASK;
1252                                 WREG32_SOC15(UVD, 0, mmUVD_DPG_PAUSE, reg_data);
1253                                 SOC15_WAIT_ON_RREG(UVD, 0, mmUVD_DPG_PAUSE,
1254                                                    UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK,
1255                                                    UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK, ret_code);
1256
1257                                 /* Restore */
1258                                 ring = &adev->vcn.ring_enc[0];
1259                                 WREG32_SOC15(UVD, 0, mmUVD_RB_BASE_LO, ring->gpu_addr);
1260                                 WREG32_SOC15(UVD, 0, mmUVD_RB_BASE_HI, upper_32_bits(ring->gpu_addr));
1261                                 WREG32_SOC15(UVD, 0, mmUVD_RB_SIZE, ring->ring_size / 4);
1262                                 WREG32_SOC15(UVD, 0, mmUVD_RB_RPTR, lower_32_bits(ring->wptr));
1263                                 WREG32_SOC15(UVD, 0, mmUVD_RB_WPTR, lower_32_bits(ring->wptr));
1264
1265                                 ring = &adev->vcn.ring_enc[1];
1266                                 WREG32_SOC15(UVD, 0, mmUVD_RB_BASE_LO2, ring->gpu_addr);
1267                                 WREG32_SOC15(UVD, 0, mmUVD_RB_BASE_HI2, upper_32_bits(ring->gpu_addr));
1268                                 WREG32_SOC15(UVD, 0, mmUVD_RB_SIZE2, ring->ring_size / 4);
1269                                 WREG32_SOC15(UVD, 0, mmUVD_RB_RPTR2, lower_32_bits(ring->wptr));
1270                                 WREG32_SOC15(UVD, 0, mmUVD_RB_WPTR2, lower_32_bits(ring->wptr));
1271
1272                                 ring = &adev->vcn.ring_dec;
1273                                 WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_WPTR,
1274                                                    RREG32_SOC15(UVD, 0, mmUVD_SCRATCH2) & 0x7FFFFFFF);
1275                                 SOC15_WAIT_ON_RREG(UVD, 0, mmUVD_POWER_STATUS,
1276                                                    UVD_PGFSM_CONFIG__UVDM_UVDU_PWR_ON,
1277                                                    UVD_POWER_STATUS__UVD_POWER_STATUS_MASK, ret_code);
1278                         }
1279                 } else {
1280                         /* unpause dpg non-jpeg, no need to wait */
1281                         reg_data &= ~UVD_DPG_PAUSE__NJ_PAUSE_DPG_REQ_MASK;
1282                         WREG32_SOC15(UVD, 0, mmUVD_DPG_PAUSE, reg_data);
1283                 }
1284                 adev->vcn.pause_state.fw_based = new_state->fw_based;
1285         }
1286
1287         /* pause/unpause if state is changed */
1288         if (adev->vcn.pause_state.jpeg != new_state->jpeg) {
1289                 DRM_DEBUG("dpg pause state changed %d:%d -> %d:%d",
1290                         adev->vcn.pause_state.fw_based, adev->vcn.pause_state.jpeg,
1291                         new_state->fw_based, new_state->jpeg);
1292
1293                 reg_data = RREG32_SOC15(UVD, 0, mmUVD_DPG_PAUSE) &
1294                         (~UVD_DPG_PAUSE__JPEG_PAUSE_DPG_ACK_MASK);
1295
1296                 if (new_state->jpeg == VCN_DPG_STATE__PAUSE) {
1297                         ret_code = 0;
1298
1299                         if (!(reg_data & UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK))
1300                                 SOC15_WAIT_ON_RREG(UVD, 0, mmUVD_POWER_STATUS,
1301                                                    UVD_POWER_STATUS__UVD_POWER_STATUS_TILES_OFF,
1302                                                    UVD_POWER_STATUS__UVD_POWER_STATUS_MASK, ret_code);
1303
1304                         if (!ret_code) {
1305                                 /* Make sure JPRG Snoop is disabled before sending the pause */
1306                                 reg_data2 = RREG32_SOC15(UVD, 0, mmUVD_POWER_STATUS);
1307                                 reg_data2 |= UVD_POWER_STATUS__JRBC_SNOOP_DIS_MASK;
1308                                 WREG32_SOC15(UVD, 0, mmUVD_POWER_STATUS, reg_data2);
1309
1310                                 /* pause DPG jpeg */
1311                                 reg_data |= UVD_DPG_PAUSE__JPEG_PAUSE_DPG_REQ_MASK;
1312                                 WREG32_SOC15(UVD, 0, mmUVD_DPG_PAUSE, reg_data);
1313                                 SOC15_WAIT_ON_RREG(UVD, 0, mmUVD_DPG_PAUSE,
1314                                                         UVD_DPG_PAUSE__JPEG_PAUSE_DPG_ACK_MASK,
1315                                                         UVD_DPG_PAUSE__JPEG_PAUSE_DPG_ACK_MASK, ret_code);
1316
1317                                 /* Restore */
1318                                 ring = &adev->vcn.ring_jpeg;
1319                                 WREG32_SOC15(UVD, 0, mmUVD_LMI_JRBC_RB_VMID, 0);
1320                                 WREG32_SOC15(UVD, 0, mmUVD_JRBC_RB_CNTL,
1321                                                         UVD_JRBC_RB_CNTL__RB_NO_FETCH_MASK |
1322                                                         UVD_JRBC_RB_CNTL__RB_RPTR_WR_EN_MASK);
1323                                 WREG32_SOC15(UVD, 0, mmUVD_LMI_JRBC_RB_64BIT_BAR_LOW,
1324                                                         lower_32_bits(ring->gpu_addr));
1325                                 WREG32_SOC15(UVD, 0, mmUVD_LMI_JRBC_RB_64BIT_BAR_HIGH,
1326                                                         upper_32_bits(ring->gpu_addr));
1327                                 WREG32_SOC15(UVD, 0, mmUVD_JRBC_RB_RPTR, ring->wptr);
1328                                 WREG32_SOC15(UVD, 0, mmUVD_JRBC_RB_WPTR, ring->wptr);
1329                                 WREG32_SOC15(UVD, 0, mmUVD_JRBC_RB_CNTL,
1330                                                         UVD_JRBC_RB_CNTL__RB_RPTR_WR_EN_MASK);
1331
1332                                 ring = &adev->vcn.ring_dec;
1333                                 WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_WPTR,
1334                                                    RREG32_SOC15(UVD, 0, mmUVD_SCRATCH2) & 0x7FFFFFFF);
1335                                 SOC15_WAIT_ON_RREG(UVD, 0, mmUVD_POWER_STATUS,
1336                                                    UVD_PGFSM_CONFIG__UVDM_UVDU_PWR_ON,
1337                                                    UVD_POWER_STATUS__UVD_POWER_STATUS_MASK, ret_code);
1338                         }
1339                 } else {
1340                         /* unpause dpg jpeg, no need to wait */
1341                         reg_data &= ~UVD_DPG_PAUSE__JPEG_PAUSE_DPG_REQ_MASK;
1342                         WREG32_SOC15(UVD, 0, mmUVD_DPG_PAUSE, reg_data);
1343                 }
1344                 adev->vcn.pause_state.jpeg = new_state->jpeg;
1345         }
1346
1347         return 0;
1348 }
1349
1350 static bool vcn_v1_0_is_idle(void *handle)
1351 {
1352         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1353
1354         return (RREG32_SOC15(VCN, 0, mmUVD_STATUS) == UVD_STATUS__IDLE);
1355 }
1356
1357 static int vcn_v1_0_wait_for_idle(void *handle)
1358 {
1359         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1360         int ret = 0;
1361
1362         SOC15_WAIT_ON_RREG(VCN, 0, mmUVD_STATUS, UVD_STATUS__IDLE,
1363                 UVD_STATUS__IDLE, ret);
1364
1365         return ret;
1366 }
1367
1368 static int vcn_v1_0_set_clockgating_state(void *handle,
1369                                           enum amd_clockgating_state state)
1370 {
1371         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1372         bool enable = (state == AMD_CG_STATE_GATE) ? true : false;
1373
1374         if (enable) {
1375                 /* wait for STATUS to clear */
1376                 if (vcn_v1_0_is_idle(handle))
1377                         return -EBUSY;
1378                 vcn_v1_0_enable_clock_gating(adev);
1379         } else {
1380                 /* disable HW gating and enable Sw gating */
1381                 vcn_v1_0_disable_clock_gating(adev);
1382         }
1383         return 0;
1384 }
1385
1386 /**
1387  * vcn_v1_0_dec_ring_get_rptr - get read pointer
1388  *
1389  * @ring: amdgpu_ring pointer
1390  *
1391  * Returns the current hardware read pointer
1392  */
1393 static uint64_t vcn_v1_0_dec_ring_get_rptr(struct amdgpu_ring *ring)
1394 {
1395         struct amdgpu_device *adev = ring->adev;
1396
1397         return RREG32_SOC15(UVD, 0, mmUVD_RBC_RB_RPTR);
1398 }
1399
1400 /**
1401  * vcn_v1_0_dec_ring_get_wptr - get write pointer
1402  *
1403  * @ring: amdgpu_ring pointer
1404  *
1405  * Returns the current hardware write pointer
1406  */
1407 static uint64_t vcn_v1_0_dec_ring_get_wptr(struct amdgpu_ring *ring)
1408 {
1409         struct amdgpu_device *adev = ring->adev;
1410
1411         return RREG32_SOC15(UVD, 0, mmUVD_RBC_RB_WPTR);
1412 }
1413
1414 /**
1415  * vcn_v1_0_dec_ring_set_wptr - set write pointer
1416  *
1417  * @ring: amdgpu_ring pointer
1418  *
1419  * Commits the write pointer to the hardware
1420  */
1421 static void vcn_v1_0_dec_ring_set_wptr(struct amdgpu_ring *ring)
1422 {
1423         struct amdgpu_device *adev = ring->adev;
1424
1425         if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG)
1426                 WREG32_SOC15(UVD, 0, mmUVD_SCRATCH2,
1427                         lower_32_bits(ring->wptr) | 0x80000000);
1428
1429         WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_WPTR, lower_32_bits(ring->wptr));
1430 }
1431
1432 /**
1433  * vcn_v1_0_dec_ring_insert_start - insert a start command
1434  *
1435  * @ring: amdgpu_ring pointer
1436  *
1437  * Write a start command to the ring.
1438  */
1439 static void vcn_v1_0_dec_ring_insert_start(struct amdgpu_ring *ring)
1440 {
1441         struct amdgpu_device *adev = ring->adev;
1442
1443         amdgpu_ring_write(ring,
1444                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_DATA0), 0));
1445         amdgpu_ring_write(ring, 0);
1446         amdgpu_ring_write(ring,
1447                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_CMD), 0));
1448         amdgpu_ring_write(ring, VCN_DEC_CMD_PACKET_START << 1);
1449 }
1450
1451 /**
1452  * vcn_v1_0_dec_ring_insert_end - insert a end command
1453  *
1454  * @ring: amdgpu_ring pointer
1455  *
1456  * Write a end command to the ring.
1457  */
1458 static void vcn_v1_0_dec_ring_insert_end(struct amdgpu_ring *ring)
1459 {
1460         struct amdgpu_device *adev = ring->adev;
1461
1462         amdgpu_ring_write(ring,
1463                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_CMD), 0));
1464         amdgpu_ring_write(ring, VCN_DEC_CMD_PACKET_END << 1);
1465 }
1466
1467 /**
1468  * vcn_v1_0_dec_ring_emit_fence - emit an fence & trap command
1469  *
1470  * @ring: amdgpu_ring pointer
1471  * @fence: fence to emit
1472  *
1473  * Write a fence and a trap command to the ring.
1474  */
1475 static void vcn_v1_0_dec_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 seq,
1476                                      unsigned flags)
1477 {
1478         struct amdgpu_device *adev = ring->adev;
1479
1480         WARN_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
1481
1482         amdgpu_ring_write(ring,
1483                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_CONTEXT_ID), 0));
1484         amdgpu_ring_write(ring, seq);
1485         amdgpu_ring_write(ring,
1486                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_DATA0), 0));
1487         amdgpu_ring_write(ring, addr & 0xffffffff);
1488         amdgpu_ring_write(ring,
1489                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_DATA1), 0));
1490         amdgpu_ring_write(ring, upper_32_bits(addr) & 0xff);
1491         amdgpu_ring_write(ring,
1492                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_CMD), 0));
1493         amdgpu_ring_write(ring, VCN_DEC_CMD_FENCE << 1);
1494
1495         amdgpu_ring_write(ring,
1496                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_DATA0), 0));
1497         amdgpu_ring_write(ring, 0);
1498         amdgpu_ring_write(ring,
1499                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_DATA1), 0));
1500         amdgpu_ring_write(ring, 0);
1501         amdgpu_ring_write(ring,
1502                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_CMD), 0));
1503         amdgpu_ring_write(ring, VCN_DEC_CMD_TRAP << 1);
1504 }
1505
1506 /**
1507  * vcn_v1_0_dec_ring_emit_ib - execute indirect buffer
1508  *
1509  * @ring: amdgpu_ring pointer
1510  * @ib: indirect buffer to execute
1511  *
1512  * Write ring commands to execute the indirect buffer
1513  */
1514 static void vcn_v1_0_dec_ring_emit_ib(struct amdgpu_ring *ring,
1515                                         struct amdgpu_job *job,
1516                                         struct amdgpu_ib *ib,
1517                                         uint32_t flags)
1518 {
1519         struct amdgpu_device *adev = ring->adev;
1520         unsigned vmid = AMDGPU_JOB_GET_VMID(job);
1521
1522         amdgpu_ring_write(ring,
1523                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_RBC_IB_VMID), 0));
1524         amdgpu_ring_write(ring, vmid);
1525
1526         amdgpu_ring_write(ring,
1527                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_RBC_IB_64BIT_BAR_LOW), 0));
1528         amdgpu_ring_write(ring, lower_32_bits(ib->gpu_addr));
1529         amdgpu_ring_write(ring,
1530                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_RBC_IB_64BIT_BAR_HIGH), 0));
1531         amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
1532         amdgpu_ring_write(ring,
1533                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_RBC_IB_SIZE), 0));
1534         amdgpu_ring_write(ring, ib->length_dw);
1535 }
1536
1537 static void vcn_v1_0_dec_ring_emit_reg_wait(struct amdgpu_ring *ring,
1538                                             uint32_t reg, uint32_t val,
1539                                             uint32_t mask)
1540 {
1541         struct amdgpu_device *adev = ring->adev;
1542
1543         amdgpu_ring_write(ring,
1544                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_DATA0), 0));
1545         amdgpu_ring_write(ring, reg << 2);
1546         amdgpu_ring_write(ring,
1547                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_DATA1), 0));
1548         amdgpu_ring_write(ring, val);
1549         amdgpu_ring_write(ring,
1550                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GP_SCRATCH8), 0));
1551         amdgpu_ring_write(ring, mask);
1552         amdgpu_ring_write(ring,
1553                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_CMD), 0));
1554         amdgpu_ring_write(ring, VCN_DEC_CMD_REG_READ_COND_WAIT << 1);
1555 }
1556
1557 static void vcn_v1_0_dec_ring_emit_vm_flush(struct amdgpu_ring *ring,
1558                                             unsigned vmid, uint64_t pd_addr)
1559 {
1560         struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->funcs->vmhub];
1561         uint32_t data0, data1, mask;
1562
1563         pd_addr = amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
1564
1565         /* wait for register write */
1566         data0 = hub->ctx0_ptb_addr_lo32 + vmid * 2;
1567         data1 = lower_32_bits(pd_addr);
1568         mask = 0xffffffff;
1569         vcn_v1_0_dec_ring_emit_reg_wait(ring, data0, data1, mask);
1570 }
1571
1572 static void vcn_v1_0_dec_ring_emit_wreg(struct amdgpu_ring *ring,
1573                                         uint32_t reg, uint32_t val)
1574 {
1575         struct amdgpu_device *adev = ring->adev;
1576
1577         amdgpu_ring_write(ring,
1578                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_DATA0), 0));
1579         amdgpu_ring_write(ring, reg << 2);
1580         amdgpu_ring_write(ring,
1581                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_DATA1), 0));
1582         amdgpu_ring_write(ring, val);
1583         amdgpu_ring_write(ring,
1584                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_CMD), 0));
1585         amdgpu_ring_write(ring, VCN_DEC_CMD_WRITE_REG << 1);
1586 }
1587
1588 /**
1589  * vcn_v1_0_enc_ring_get_rptr - get enc read pointer
1590  *
1591  * @ring: amdgpu_ring pointer
1592  *
1593  * Returns the current hardware enc read pointer
1594  */
1595 static uint64_t vcn_v1_0_enc_ring_get_rptr(struct amdgpu_ring *ring)
1596 {
1597         struct amdgpu_device *adev = ring->adev;
1598
1599         if (ring == &adev->vcn.ring_enc[0])
1600                 return RREG32_SOC15(UVD, 0, mmUVD_RB_RPTR);
1601         else
1602                 return RREG32_SOC15(UVD, 0, mmUVD_RB_RPTR2);
1603 }
1604
1605  /**
1606  * vcn_v1_0_enc_ring_get_wptr - get enc write pointer
1607  *
1608  * @ring: amdgpu_ring pointer
1609  *
1610  * Returns the current hardware enc write pointer
1611  */
1612 static uint64_t vcn_v1_0_enc_ring_get_wptr(struct amdgpu_ring *ring)
1613 {
1614         struct amdgpu_device *adev = ring->adev;
1615
1616         if (ring == &adev->vcn.ring_enc[0])
1617                 return RREG32_SOC15(UVD, 0, mmUVD_RB_WPTR);
1618         else
1619                 return RREG32_SOC15(UVD, 0, mmUVD_RB_WPTR2);
1620 }
1621
1622  /**
1623  * vcn_v1_0_enc_ring_set_wptr - set enc write pointer
1624  *
1625  * @ring: amdgpu_ring pointer
1626  *
1627  * Commits the enc write pointer to the hardware
1628  */
1629 static void vcn_v1_0_enc_ring_set_wptr(struct amdgpu_ring *ring)
1630 {
1631         struct amdgpu_device *adev = ring->adev;
1632
1633         if (ring == &adev->vcn.ring_enc[0])
1634                 WREG32_SOC15(UVD, 0, mmUVD_RB_WPTR,
1635                         lower_32_bits(ring->wptr));
1636         else
1637                 WREG32_SOC15(UVD, 0, mmUVD_RB_WPTR2,
1638                         lower_32_bits(ring->wptr));
1639 }
1640
1641 /**
1642  * vcn_v1_0_enc_ring_emit_fence - emit an enc fence & trap command
1643  *
1644  * @ring: amdgpu_ring pointer
1645  * @fence: fence to emit
1646  *
1647  * Write enc a fence and a trap command to the ring.
1648  */
1649 static void vcn_v1_0_enc_ring_emit_fence(struct amdgpu_ring *ring, u64 addr,
1650                         u64 seq, unsigned flags)
1651 {
1652         WARN_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
1653
1654         amdgpu_ring_write(ring, VCN_ENC_CMD_FENCE);
1655         amdgpu_ring_write(ring, addr);
1656         amdgpu_ring_write(ring, upper_32_bits(addr));
1657         amdgpu_ring_write(ring, seq);
1658         amdgpu_ring_write(ring, VCN_ENC_CMD_TRAP);
1659 }
1660
1661 static void vcn_v1_0_enc_ring_insert_end(struct amdgpu_ring *ring)
1662 {
1663         amdgpu_ring_write(ring, VCN_ENC_CMD_END);
1664 }
1665
1666 /**
1667  * vcn_v1_0_enc_ring_emit_ib - enc execute indirect buffer
1668  *
1669  * @ring: amdgpu_ring pointer
1670  * @ib: indirect buffer to execute
1671  *
1672  * Write enc ring commands to execute the indirect buffer
1673  */
1674 static void vcn_v1_0_enc_ring_emit_ib(struct amdgpu_ring *ring,
1675                                         struct amdgpu_job *job,
1676                                         struct amdgpu_ib *ib,
1677                                         uint32_t flags)
1678 {
1679         unsigned vmid = AMDGPU_JOB_GET_VMID(job);
1680
1681         amdgpu_ring_write(ring, VCN_ENC_CMD_IB);
1682         amdgpu_ring_write(ring, vmid);
1683         amdgpu_ring_write(ring, lower_32_bits(ib->gpu_addr));
1684         amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
1685         amdgpu_ring_write(ring, ib->length_dw);
1686 }
1687
1688 static void vcn_v1_0_enc_ring_emit_reg_wait(struct amdgpu_ring *ring,
1689                                             uint32_t reg, uint32_t val,
1690                                             uint32_t mask)
1691 {
1692         amdgpu_ring_write(ring, VCN_ENC_CMD_REG_WAIT);
1693         amdgpu_ring_write(ring, reg << 2);
1694         amdgpu_ring_write(ring, mask);
1695         amdgpu_ring_write(ring, val);
1696 }
1697
1698 static void vcn_v1_0_enc_ring_emit_vm_flush(struct amdgpu_ring *ring,
1699                                             unsigned int vmid, uint64_t pd_addr)
1700 {
1701         struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->funcs->vmhub];
1702
1703         pd_addr = amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
1704
1705         /* wait for reg writes */
1706         vcn_v1_0_enc_ring_emit_reg_wait(ring, hub->ctx0_ptb_addr_lo32 + vmid * 2,
1707                                         lower_32_bits(pd_addr), 0xffffffff);
1708 }
1709
1710 static void vcn_v1_0_enc_ring_emit_wreg(struct amdgpu_ring *ring,
1711                                         uint32_t reg, uint32_t val)
1712 {
1713         amdgpu_ring_write(ring, VCN_ENC_CMD_REG_WRITE);
1714         amdgpu_ring_write(ring, reg << 2);
1715         amdgpu_ring_write(ring, val);
1716 }
1717
1718
1719 /**
1720  * vcn_v1_0_jpeg_ring_get_rptr - get read pointer
1721  *
1722  * @ring: amdgpu_ring pointer
1723  *
1724  * Returns the current hardware read pointer
1725  */
1726 static uint64_t vcn_v1_0_jpeg_ring_get_rptr(struct amdgpu_ring *ring)
1727 {
1728         struct amdgpu_device *adev = ring->adev;
1729
1730         return RREG32_SOC15(UVD, 0, mmUVD_JRBC_RB_RPTR);
1731 }
1732
1733 /**
1734  * vcn_v1_0_jpeg_ring_get_wptr - get write pointer
1735  *
1736  * @ring: amdgpu_ring pointer
1737  *
1738  * Returns the current hardware write pointer
1739  */
1740 static uint64_t vcn_v1_0_jpeg_ring_get_wptr(struct amdgpu_ring *ring)
1741 {
1742         struct amdgpu_device *adev = ring->adev;
1743
1744         return RREG32_SOC15(UVD, 0, mmUVD_JRBC_RB_WPTR);
1745 }
1746
1747 /**
1748  * vcn_v1_0_jpeg_ring_set_wptr - set write pointer
1749  *
1750  * @ring: amdgpu_ring pointer
1751  *
1752  * Commits the write pointer to the hardware
1753  */
1754 static void vcn_v1_0_jpeg_ring_set_wptr(struct amdgpu_ring *ring)
1755 {
1756         struct amdgpu_device *adev = ring->adev;
1757
1758         WREG32_SOC15(UVD, 0, mmUVD_JRBC_RB_WPTR, lower_32_bits(ring->wptr));
1759 }
1760
1761 /**
1762  * vcn_v1_0_jpeg_ring_insert_start - insert a start command
1763  *
1764  * @ring: amdgpu_ring pointer
1765  *
1766  * Write a start command to the ring.
1767  */
1768 static void vcn_v1_0_jpeg_ring_insert_start(struct amdgpu_ring *ring)
1769 {
1770         struct amdgpu_device *adev = ring->adev;
1771
1772         amdgpu_ring_write(ring,
1773                 PACKETJ(SOC15_REG_OFFSET(UVD, 0, mmUVD_JRBC_EXTERNAL_REG_BASE), 0, 0, PACKETJ_TYPE0));
1774         amdgpu_ring_write(ring, 0x68e04);
1775
1776         amdgpu_ring_write(ring, PACKETJ(0, 0, 0, PACKETJ_TYPE0));
1777         amdgpu_ring_write(ring, 0x80010000);
1778 }
1779
1780 /**
1781  * vcn_v1_0_jpeg_ring_insert_end - insert a end command
1782  *
1783  * @ring: amdgpu_ring pointer
1784  *
1785  * Write a end command to the ring.
1786  */
1787 static void vcn_v1_0_jpeg_ring_insert_end(struct amdgpu_ring *ring)
1788 {
1789         struct amdgpu_device *adev = ring->adev;
1790
1791         amdgpu_ring_write(ring,
1792                 PACKETJ(SOC15_REG_OFFSET(UVD, 0, mmUVD_JRBC_EXTERNAL_REG_BASE), 0, 0, PACKETJ_TYPE0));
1793         amdgpu_ring_write(ring, 0x68e04);
1794
1795         amdgpu_ring_write(ring, PACKETJ(0, 0, 0, PACKETJ_TYPE0));
1796         amdgpu_ring_write(ring, 0x00010000);
1797 }
1798
1799 /**
1800  * vcn_v1_0_jpeg_ring_emit_fence - emit an fence & trap command
1801  *
1802  * @ring: amdgpu_ring pointer
1803  * @fence: fence to emit
1804  *
1805  * Write a fence and a trap command to the ring.
1806  */
1807 static void vcn_v1_0_jpeg_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 seq,
1808                                      unsigned flags)
1809 {
1810         struct amdgpu_device *adev = ring->adev;
1811
1812         WARN_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
1813
1814         amdgpu_ring_write(ring,
1815                 PACKETJ(SOC15_REG_OFFSET(UVD, 0, mmUVD_JPEG_GPCOM_DATA0), 0, 0, PACKETJ_TYPE0));
1816         amdgpu_ring_write(ring, seq);
1817
1818         amdgpu_ring_write(ring,
1819                 PACKETJ(SOC15_REG_OFFSET(UVD, 0, mmUVD_JPEG_GPCOM_DATA1), 0, 0, PACKETJ_TYPE0));
1820         amdgpu_ring_write(ring, seq);
1821
1822         amdgpu_ring_write(ring,
1823                 PACKETJ(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_JRBC_RB_MEM_WR_64BIT_BAR_LOW), 0, 0, PACKETJ_TYPE0));
1824         amdgpu_ring_write(ring, lower_32_bits(addr));
1825
1826         amdgpu_ring_write(ring,
1827                 PACKETJ(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_JRBC_RB_MEM_WR_64BIT_BAR_HIGH), 0, 0, PACKETJ_TYPE0));
1828         amdgpu_ring_write(ring, upper_32_bits(addr));
1829
1830         amdgpu_ring_write(ring,
1831                 PACKETJ(SOC15_REG_OFFSET(UVD, 0, mmUVD_JPEG_GPCOM_CMD), 0, 0, PACKETJ_TYPE0));
1832         amdgpu_ring_write(ring, 0x8);
1833
1834         amdgpu_ring_write(ring,
1835                 PACKETJ(SOC15_REG_OFFSET(UVD, 0, mmUVD_JPEG_GPCOM_CMD), 0, PACKETJ_CONDITION_CHECK0, PACKETJ_TYPE4));
1836         amdgpu_ring_write(ring, 0);
1837
1838         amdgpu_ring_write(ring,
1839                 PACKETJ(SOC15_REG_OFFSET(UVD, 0, mmUVD_JRBC_RB_COND_RD_TIMER), 0, 0, PACKETJ_TYPE0));
1840         amdgpu_ring_write(ring, 0x01400200);
1841
1842         amdgpu_ring_write(ring,
1843                 PACKETJ(SOC15_REG_OFFSET(UVD, 0, mmUVD_JRBC_RB_REF_DATA), 0, 0, PACKETJ_TYPE0));
1844         amdgpu_ring_write(ring, seq);
1845
1846         amdgpu_ring_write(ring,
1847                 PACKETJ(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_JRBC_RB_MEM_RD_64BIT_BAR_LOW), 0, 0, PACKETJ_TYPE0));
1848         amdgpu_ring_write(ring, lower_32_bits(addr));
1849
1850         amdgpu_ring_write(ring,
1851                 PACKETJ(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_JRBC_RB_MEM_RD_64BIT_BAR_HIGH), 0, 0, PACKETJ_TYPE0));
1852         amdgpu_ring_write(ring, upper_32_bits(addr));
1853
1854         amdgpu_ring_write(ring,
1855                 PACKETJ(0, 0, PACKETJ_CONDITION_CHECK3, PACKETJ_TYPE2));
1856         amdgpu_ring_write(ring, 0xffffffff);
1857
1858         amdgpu_ring_write(ring,
1859                 PACKETJ(SOC15_REG_OFFSET(UVD, 0, mmUVD_JRBC_EXTERNAL_REG_BASE), 0, 0, PACKETJ_TYPE0));
1860         amdgpu_ring_write(ring, 0x3fbc);
1861
1862         amdgpu_ring_write(ring,
1863                 PACKETJ(0, 0, 0, PACKETJ_TYPE0));
1864         amdgpu_ring_write(ring, 0x1);
1865
1866         /* emit trap */
1867         amdgpu_ring_write(ring, PACKETJ(0, 0, 0, PACKETJ_TYPE7));
1868         amdgpu_ring_write(ring, 0);
1869 }
1870
1871 /**
1872  * vcn_v1_0_jpeg_ring_emit_ib - execute indirect buffer
1873  *
1874  * @ring: amdgpu_ring pointer
1875  * @ib: indirect buffer to execute
1876  *
1877  * Write ring commands to execute the indirect buffer.
1878  */
1879 static void vcn_v1_0_jpeg_ring_emit_ib(struct amdgpu_ring *ring,
1880                                         struct amdgpu_job *job,
1881                                         struct amdgpu_ib *ib,
1882                                         uint32_t flags)
1883 {
1884         struct amdgpu_device *adev = ring->adev;
1885         unsigned vmid = AMDGPU_JOB_GET_VMID(job);
1886
1887         amdgpu_ring_write(ring,
1888                 PACKETJ(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_JRBC_IB_VMID), 0, 0, PACKETJ_TYPE0));
1889         amdgpu_ring_write(ring, (vmid | (vmid << 4)));
1890
1891         amdgpu_ring_write(ring,
1892                 PACKETJ(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_JPEG_VMID), 0, 0, PACKETJ_TYPE0));
1893         amdgpu_ring_write(ring, (vmid | (vmid << 4)));
1894
1895         amdgpu_ring_write(ring,
1896                 PACKETJ(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_JRBC_IB_64BIT_BAR_LOW), 0, 0, PACKETJ_TYPE0));
1897         amdgpu_ring_write(ring, lower_32_bits(ib->gpu_addr));
1898
1899         amdgpu_ring_write(ring,
1900                 PACKETJ(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_JRBC_IB_64BIT_BAR_HIGH), 0, 0, PACKETJ_TYPE0));
1901         amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
1902
1903         amdgpu_ring_write(ring,
1904                 PACKETJ(SOC15_REG_OFFSET(UVD, 0, mmUVD_JRBC_IB_SIZE), 0, 0, PACKETJ_TYPE0));
1905         amdgpu_ring_write(ring, ib->length_dw);
1906
1907         amdgpu_ring_write(ring,
1908                 PACKETJ(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_JRBC_RB_MEM_RD_64BIT_BAR_LOW), 0, 0, PACKETJ_TYPE0));
1909         amdgpu_ring_write(ring, lower_32_bits(ring->gpu_addr));
1910
1911         amdgpu_ring_write(ring,
1912                 PACKETJ(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_JRBC_RB_MEM_RD_64BIT_BAR_HIGH), 0, 0, PACKETJ_TYPE0));
1913         amdgpu_ring_write(ring, upper_32_bits(ring->gpu_addr));
1914
1915         amdgpu_ring_write(ring,
1916                 PACKETJ(0, 0, PACKETJ_CONDITION_CHECK0, PACKETJ_TYPE2));
1917         amdgpu_ring_write(ring, 0);
1918
1919         amdgpu_ring_write(ring,
1920                 PACKETJ(SOC15_REG_OFFSET(UVD, 0, mmUVD_JRBC_RB_COND_RD_TIMER), 0, 0, PACKETJ_TYPE0));
1921         amdgpu_ring_write(ring, 0x01400200);
1922
1923         amdgpu_ring_write(ring,
1924                 PACKETJ(SOC15_REG_OFFSET(UVD, 0, mmUVD_JRBC_RB_REF_DATA), 0, 0, PACKETJ_TYPE0));
1925         amdgpu_ring_write(ring, 0x2);
1926
1927         amdgpu_ring_write(ring,
1928                 PACKETJ(SOC15_REG_OFFSET(UVD, 0, mmUVD_JRBC_STATUS), 0, PACKETJ_CONDITION_CHECK3, PACKETJ_TYPE3));
1929         amdgpu_ring_write(ring, 0x2);
1930 }
1931
1932 static void vcn_v1_0_jpeg_ring_emit_reg_wait(struct amdgpu_ring *ring,
1933                                             uint32_t reg, uint32_t val,
1934                                             uint32_t mask)
1935 {
1936         struct amdgpu_device *adev = ring->adev;
1937         uint32_t reg_offset = (reg << 2);
1938
1939         amdgpu_ring_write(ring,
1940                 PACKETJ(SOC15_REG_OFFSET(UVD, 0, mmUVD_JRBC_RB_COND_RD_TIMER), 0, 0, PACKETJ_TYPE0));
1941         amdgpu_ring_write(ring, 0x01400200);
1942
1943         amdgpu_ring_write(ring,
1944                 PACKETJ(SOC15_REG_OFFSET(UVD, 0, mmUVD_JRBC_RB_REF_DATA), 0, 0, PACKETJ_TYPE0));
1945         amdgpu_ring_write(ring, val);
1946
1947         amdgpu_ring_write(ring,
1948                 PACKETJ(SOC15_REG_OFFSET(UVD, 0, mmUVD_JRBC_EXTERNAL_REG_BASE), 0, 0, PACKETJ_TYPE0));
1949         if (((reg_offset >= 0x1f800) && (reg_offset <= 0x21fff)) ||
1950                 ((reg_offset >= 0x1e000) && (reg_offset <= 0x1e1ff))) {
1951                 amdgpu_ring_write(ring, 0);
1952                 amdgpu_ring_write(ring,
1953                         PACKETJ((reg_offset >> 2), 0, 0, PACKETJ_TYPE3));
1954         } else {
1955                 amdgpu_ring_write(ring, reg_offset);
1956                 amdgpu_ring_write(ring,
1957                         PACKETJ(0, 0, 0, PACKETJ_TYPE3));
1958         }
1959         amdgpu_ring_write(ring, mask);
1960 }
1961
1962 static void vcn_v1_0_jpeg_ring_emit_vm_flush(struct amdgpu_ring *ring,
1963                 unsigned vmid, uint64_t pd_addr)
1964 {
1965         struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->funcs->vmhub];
1966         uint32_t data0, data1, mask;
1967
1968         pd_addr = amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
1969
1970         /* wait for register write */
1971         data0 = hub->ctx0_ptb_addr_lo32 + vmid * 2;
1972         data1 = lower_32_bits(pd_addr);
1973         mask = 0xffffffff;
1974         vcn_v1_0_jpeg_ring_emit_reg_wait(ring, data0, data1, mask);
1975 }
1976
1977 static void vcn_v1_0_jpeg_ring_emit_wreg(struct amdgpu_ring *ring,
1978                                         uint32_t reg, uint32_t val)
1979 {
1980         struct amdgpu_device *adev = ring->adev;
1981         uint32_t reg_offset = (reg << 2);
1982
1983         amdgpu_ring_write(ring,
1984                 PACKETJ(SOC15_REG_OFFSET(UVD, 0, mmUVD_JRBC_EXTERNAL_REG_BASE), 0, 0, PACKETJ_TYPE0));
1985         if (((reg_offset >= 0x1f800) && (reg_offset <= 0x21fff)) ||
1986                         ((reg_offset >= 0x1e000) && (reg_offset <= 0x1e1ff))) {
1987                 amdgpu_ring_write(ring, 0);
1988                 amdgpu_ring_write(ring,
1989                         PACKETJ((reg_offset >> 2), 0, 0, PACKETJ_TYPE0));
1990         } else {
1991                 amdgpu_ring_write(ring, reg_offset);
1992                 amdgpu_ring_write(ring,
1993                         PACKETJ(0, 0, 0, PACKETJ_TYPE0));
1994         }
1995         amdgpu_ring_write(ring, val);
1996 }
1997
1998 static void vcn_v1_0_jpeg_ring_nop(struct amdgpu_ring *ring, uint32_t count)
1999 {
2000         int i;
2001
2002         WARN_ON(ring->wptr % 2 || count % 2);
2003
2004         for (i = 0; i < count / 2; i++) {
2005                 amdgpu_ring_write(ring, PACKETJ(0, 0, 0, PACKETJ_TYPE6));
2006                 amdgpu_ring_write(ring, 0);
2007         }
2008 }
2009
2010 static void vcn_v1_0_jpeg_ring_patch_wreg(struct amdgpu_ring *ring, uint32_t *ptr, uint32_t reg_offset, uint32_t val)
2011 {
2012         struct amdgpu_device *adev = ring->adev;
2013         ring->ring[(*ptr)++] = PACKETJ(SOC15_REG_OFFSET(UVD, 0, mmUVD_JRBC_EXTERNAL_REG_BASE), 0, 0, PACKETJ_TYPE0);
2014         if (((reg_offset >= 0x1f800) && (reg_offset <= 0x21fff)) ||
2015                 ((reg_offset >= 0x1e000) && (reg_offset <= 0x1e1ff))) {
2016                 ring->ring[(*ptr)++] = 0;
2017                 ring->ring[(*ptr)++] = PACKETJ((reg_offset >> 2), 0, 0, PACKETJ_TYPE0);
2018         } else {
2019                 ring->ring[(*ptr)++] = reg_offset;
2020                 ring->ring[(*ptr)++] = PACKETJ(0, 0, 0, PACKETJ_TYPE0);
2021         }
2022         ring->ring[(*ptr)++] = val;
2023 }
2024
2025 static void vcn_v1_0_jpeg_ring_set_patch_ring(struct amdgpu_ring *ring, uint32_t ptr)
2026 {
2027         struct amdgpu_device *adev = ring->adev;
2028
2029         uint32_t reg, reg_offset, val, mask, i;
2030
2031         // 1st: program mmUVD_LMI_JRBC_RB_MEM_RD_64BIT_BAR_LOW
2032         reg = SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_JRBC_RB_MEM_RD_64BIT_BAR_LOW);
2033         reg_offset = (reg << 2);
2034         val = lower_32_bits(ring->gpu_addr);
2035         vcn_v1_0_jpeg_ring_patch_wreg(ring, &ptr, reg_offset, val);
2036
2037         // 2nd: program mmUVD_LMI_JRBC_RB_MEM_RD_64BIT_BAR_HIGH
2038         reg = SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_JRBC_RB_MEM_RD_64BIT_BAR_HIGH);
2039         reg_offset = (reg << 2);
2040         val = upper_32_bits(ring->gpu_addr);
2041         vcn_v1_0_jpeg_ring_patch_wreg(ring, &ptr, reg_offset, val);
2042
2043         // 3rd to 5th: issue MEM_READ commands
2044         for (i = 0; i <= 2; i++) {
2045                 ring->ring[ptr++] = PACKETJ(0, 0, 0, PACKETJ_TYPE2);
2046                 ring->ring[ptr++] = 0;
2047         }
2048
2049         // 6th: program mmUVD_JRBC_RB_CNTL register to enable NO_FETCH and RPTR write ability
2050         reg = SOC15_REG_OFFSET(UVD, 0, mmUVD_JRBC_RB_CNTL);
2051         reg_offset = (reg << 2);
2052         val = 0x13;
2053         vcn_v1_0_jpeg_ring_patch_wreg(ring, &ptr, reg_offset, val);
2054
2055         // 7th: program mmUVD_JRBC_RB_REF_DATA
2056         reg = SOC15_REG_OFFSET(UVD, 0, mmUVD_JRBC_RB_REF_DATA);
2057         reg_offset = (reg << 2);
2058         val = 0x1;
2059         vcn_v1_0_jpeg_ring_patch_wreg(ring, &ptr, reg_offset, val);
2060
2061         // 8th: issue conditional register read mmUVD_JRBC_RB_CNTL
2062         reg = SOC15_REG_OFFSET(UVD, 0, mmUVD_JRBC_RB_CNTL);
2063         reg_offset = (reg << 2);
2064         val = 0x1;
2065         mask = 0x1;
2066
2067         ring->ring[ptr++] = PACKETJ(SOC15_REG_OFFSET(UVD, 0, mmUVD_JRBC_RB_COND_RD_TIMER), 0, 0, PACKETJ_TYPE0);
2068         ring->ring[ptr++] = 0x01400200;
2069         ring->ring[ptr++] = PACKETJ(SOC15_REG_OFFSET(UVD, 0, mmUVD_JRBC_RB_REF_DATA), 0, 0, PACKETJ_TYPE0);
2070         ring->ring[ptr++] = val;
2071         ring->ring[ptr++] = PACKETJ(SOC15_REG_OFFSET(UVD, 0, mmUVD_JRBC_EXTERNAL_REG_BASE), 0, 0, PACKETJ_TYPE0);
2072         if (((reg_offset >= 0x1f800) && (reg_offset <= 0x21fff)) ||
2073                 ((reg_offset >= 0x1e000) && (reg_offset <= 0x1e1ff))) {
2074                 ring->ring[ptr++] = 0;
2075                 ring->ring[ptr++] = PACKETJ((reg_offset >> 2), 0, 0, PACKETJ_TYPE3);
2076         } else {
2077                 ring->ring[ptr++] = reg_offset;
2078                 ring->ring[ptr++] = PACKETJ(0, 0, 0, PACKETJ_TYPE3);
2079         }
2080         ring->ring[ptr++] = mask;
2081
2082         //9th to 21st: insert no-op
2083         for (i = 0; i <= 12; i++) {
2084                 ring->ring[ptr++] = PACKETJ(0, 0, 0, PACKETJ_TYPE6);
2085                 ring->ring[ptr++] = 0;
2086         }
2087
2088         //22nd: reset mmUVD_JRBC_RB_RPTR
2089         reg = SOC15_REG_OFFSET(UVD, 0, mmUVD_JRBC_RB_RPTR);
2090         reg_offset = (reg << 2);
2091         val = 0;
2092         vcn_v1_0_jpeg_ring_patch_wreg(ring, &ptr, reg_offset, val);
2093
2094         //23rd: program mmUVD_JRBC_RB_CNTL to disable no_fetch
2095         reg = SOC15_REG_OFFSET(UVD, 0, mmUVD_JRBC_RB_CNTL);
2096         reg_offset = (reg << 2);
2097         val = 0x12;
2098         vcn_v1_0_jpeg_ring_patch_wreg(ring, &ptr, reg_offset, val);
2099 }
2100
2101 static int vcn_v1_0_set_interrupt_state(struct amdgpu_device *adev,
2102                                         struct amdgpu_irq_src *source,
2103                                         unsigned type,
2104                                         enum amdgpu_interrupt_state state)
2105 {
2106         return 0;
2107 }
2108
2109 static int vcn_v1_0_process_interrupt(struct amdgpu_device *adev,
2110                                       struct amdgpu_irq_src *source,
2111                                       struct amdgpu_iv_entry *entry)
2112 {
2113         DRM_DEBUG("IH: VCN TRAP\n");
2114
2115         switch (entry->src_id) {
2116         case 124:
2117                 amdgpu_fence_process(&adev->vcn.ring_dec);
2118                 break;
2119         case 119:
2120                 amdgpu_fence_process(&adev->vcn.ring_enc[0]);
2121                 break;
2122         case 120:
2123                 amdgpu_fence_process(&adev->vcn.ring_enc[1]);
2124                 break;
2125         case 126:
2126                 amdgpu_fence_process(&adev->vcn.ring_jpeg);
2127                 break;
2128         default:
2129                 DRM_ERROR("Unhandled interrupt: %d %d\n",
2130                           entry->src_id, entry->src_data[0]);
2131                 break;
2132         }
2133
2134         return 0;
2135 }
2136
2137 static void vcn_v1_0_dec_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count)
2138 {
2139         struct amdgpu_device *adev = ring->adev;
2140         int i;
2141
2142         WARN_ON(ring->wptr % 2 || count % 2);
2143
2144         for (i = 0; i < count / 2; i++) {
2145                 amdgpu_ring_write(ring, PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_NO_OP), 0));
2146                 amdgpu_ring_write(ring, 0);
2147         }
2148 }
2149
2150 static int vcn_v1_0_set_powergating_state(void *handle,
2151                                           enum amd_powergating_state state)
2152 {
2153         /* This doesn't actually powergate the VCN block.
2154          * That's done in the dpm code via the SMC.  This
2155          * just re-inits the block as necessary.  The actual
2156          * gating still happens in the dpm code.  We should
2157          * revisit this when there is a cleaner line between
2158          * the smc and the hw blocks
2159          */
2160         int ret;
2161         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2162
2163         if(state == adev->vcn.cur_state)
2164                 return 0;
2165
2166         if (state == AMD_PG_STATE_GATE)
2167                 ret = vcn_v1_0_stop(adev);
2168         else
2169                 ret = vcn_v1_0_start(adev);
2170
2171         if(!ret)
2172                 adev->vcn.cur_state = state;
2173         return ret;
2174 }
2175
2176 static const struct amd_ip_funcs vcn_v1_0_ip_funcs = {
2177         .name = "vcn_v1_0",
2178         .early_init = vcn_v1_0_early_init,
2179         .late_init = NULL,
2180         .sw_init = vcn_v1_0_sw_init,
2181         .sw_fini = vcn_v1_0_sw_fini,
2182         .hw_init = vcn_v1_0_hw_init,
2183         .hw_fini = vcn_v1_0_hw_fini,
2184         .suspend = vcn_v1_0_suspend,
2185         .resume = vcn_v1_0_resume,
2186         .is_idle = vcn_v1_0_is_idle,
2187         .wait_for_idle = vcn_v1_0_wait_for_idle,
2188         .check_soft_reset = NULL /* vcn_v1_0_check_soft_reset */,
2189         .pre_soft_reset = NULL /* vcn_v1_0_pre_soft_reset */,
2190         .soft_reset = NULL /* vcn_v1_0_soft_reset */,
2191         .post_soft_reset = NULL /* vcn_v1_0_post_soft_reset */,
2192         .set_clockgating_state = vcn_v1_0_set_clockgating_state,
2193         .set_powergating_state = vcn_v1_0_set_powergating_state,
2194 };
2195
2196 static const struct amdgpu_ring_funcs vcn_v1_0_dec_ring_vm_funcs = {
2197         .type = AMDGPU_RING_TYPE_VCN_DEC,
2198         .align_mask = 0xf,
2199         .support_64bit_ptrs = false,
2200         .no_user_fence = true,
2201         .vmhub = AMDGPU_MMHUB,
2202         .get_rptr = vcn_v1_0_dec_ring_get_rptr,
2203         .get_wptr = vcn_v1_0_dec_ring_get_wptr,
2204         .set_wptr = vcn_v1_0_dec_ring_set_wptr,
2205         .emit_frame_size =
2206                 6 + 6 + /* hdp invalidate / flush */
2207                 SOC15_FLUSH_GPU_TLB_NUM_WREG * 6 +
2208                 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 8 +
2209                 8 + /* vcn_v1_0_dec_ring_emit_vm_flush */
2210                 14 + 14 + /* vcn_v1_0_dec_ring_emit_fence x2 vm fence */
2211                 6,
2212         .emit_ib_size = 8, /* vcn_v1_0_dec_ring_emit_ib */
2213         .emit_ib = vcn_v1_0_dec_ring_emit_ib,
2214         .emit_fence = vcn_v1_0_dec_ring_emit_fence,
2215         .emit_vm_flush = vcn_v1_0_dec_ring_emit_vm_flush,
2216         .test_ring = amdgpu_vcn_dec_ring_test_ring,
2217         .test_ib = amdgpu_vcn_dec_ring_test_ib,
2218         .insert_nop = vcn_v1_0_dec_ring_insert_nop,
2219         .insert_start = vcn_v1_0_dec_ring_insert_start,
2220         .insert_end = vcn_v1_0_dec_ring_insert_end,
2221         .pad_ib = amdgpu_ring_generic_pad_ib,
2222         .begin_use = amdgpu_vcn_ring_begin_use,
2223         .end_use = amdgpu_vcn_ring_end_use,
2224         .emit_wreg = vcn_v1_0_dec_ring_emit_wreg,
2225         .emit_reg_wait = vcn_v1_0_dec_ring_emit_reg_wait,
2226         .emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
2227 };
2228
2229 static const struct amdgpu_ring_funcs vcn_v1_0_enc_ring_vm_funcs = {
2230         .type = AMDGPU_RING_TYPE_VCN_ENC,
2231         .align_mask = 0x3f,
2232         .nop = VCN_ENC_CMD_NO_OP,
2233         .support_64bit_ptrs = false,
2234         .no_user_fence = true,
2235         .vmhub = AMDGPU_MMHUB,
2236         .get_rptr = vcn_v1_0_enc_ring_get_rptr,
2237         .get_wptr = vcn_v1_0_enc_ring_get_wptr,
2238         .set_wptr = vcn_v1_0_enc_ring_set_wptr,
2239         .emit_frame_size =
2240                 SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 +
2241                 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 4 +
2242                 4 + /* vcn_v1_0_enc_ring_emit_vm_flush */
2243                 5 + 5 + /* vcn_v1_0_enc_ring_emit_fence x2 vm fence */
2244                 1, /* vcn_v1_0_enc_ring_insert_end */
2245         .emit_ib_size = 5, /* vcn_v1_0_enc_ring_emit_ib */
2246         .emit_ib = vcn_v1_0_enc_ring_emit_ib,
2247         .emit_fence = vcn_v1_0_enc_ring_emit_fence,
2248         .emit_vm_flush = vcn_v1_0_enc_ring_emit_vm_flush,
2249         .test_ring = amdgpu_vcn_enc_ring_test_ring,
2250         .test_ib = amdgpu_vcn_enc_ring_test_ib,
2251         .insert_nop = amdgpu_ring_insert_nop,
2252         .insert_end = vcn_v1_0_enc_ring_insert_end,
2253         .pad_ib = amdgpu_ring_generic_pad_ib,
2254         .begin_use = amdgpu_vcn_ring_begin_use,
2255         .end_use = amdgpu_vcn_ring_end_use,
2256         .emit_wreg = vcn_v1_0_enc_ring_emit_wreg,
2257         .emit_reg_wait = vcn_v1_0_enc_ring_emit_reg_wait,
2258         .emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
2259 };
2260
2261 static const struct amdgpu_ring_funcs vcn_v1_0_jpeg_ring_vm_funcs = {
2262         .type = AMDGPU_RING_TYPE_VCN_JPEG,
2263         .align_mask = 0xf,
2264         .nop = PACKET0(0x81ff, 0),
2265         .support_64bit_ptrs = false,
2266         .no_user_fence = true,
2267         .vmhub = AMDGPU_MMHUB,
2268         .extra_dw = 64,
2269         .get_rptr = vcn_v1_0_jpeg_ring_get_rptr,
2270         .get_wptr = vcn_v1_0_jpeg_ring_get_wptr,
2271         .set_wptr = vcn_v1_0_jpeg_ring_set_wptr,
2272         .emit_frame_size =
2273                 6 + 6 + /* hdp invalidate / flush */
2274                 SOC15_FLUSH_GPU_TLB_NUM_WREG * 6 +
2275                 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 8 +
2276                 8 + /* vcn_v1_0_jpeg_ring_emit_vm_flush */
2277                 26 + 26 + /* vcn_v1_0_jpeg_ring_emit_fence x2 vm fence */
2278                 6,
2279         .emit_ib_size = 22, /* vcn_v1_0_jpeg_ring_emit_ib */
2280         .emit_ib = vcn_v1_0_jpeg_ring_emit_ib,
2281         .emit_fence = vcn_v1_0_jpeg_ring_emit_fence,
2282         .emit_vm_flush = vcn_v1_0_jpeg_ring_emit_vm_flush,
2283         .test_ring = amdgpu_vcn_jpeg_ring_test_ring,
2284         .test_ib = amdgpu_vcn_jpeg_ring_test_ib,
2285         .insert_nop = vcn_v1_0_jpeg_ring_nop,
2286         .insert_start = vcn_v1_0_jpeg_ring_insert_start,
2287         .insert_end = vcn_v1_0_jpeg_ring_insert_end,
2288         .pad_ib = amdgpu_ring_generic_pad_ib,
2289         .begin_use = amdgpu_vcn_ring_begin_use,
2290         .end_use = amdgpu_vcn_ring_end_use,
2291         .emit_wreg = vcn_v1_0_jpeg_ring_emit_wreg,
2292         .emit_reg_wait = vcn_v1_0_jpeg_ring_emit_reg_wait,
2293         .emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
2294 };
2295
2296 static void vcn_v1_0_set_dec_ring_funcs(struct amdgpu_device *adev)
2297 {
2298         adev->vcn.ring_dec.funcs = &vcn_v1_0_dec_ring_vm_funcs;
2299         DRM_INFO("VCN decode is enabled in VM mode\n");
2300 }
2301
2302 static void vcn_v1_0_set_enc_ring_funcs(struct amdgpu_device *adev)
2303 {
2304         int i;
2305
2306         for (i = 0; i < adev->vcn.num_enc_rings; ++i)
2307                 adev->vcn.ring_enc[i].funcs = &vcn_v1_0_enc_ring_vm_funcs;
2308
2309         DRM_INFO("VCN encode is enabled in VM mode\n");
2310 }
2311
2312 static void vcn_v1_0_set_jpeg_ring_funcs(struct amdgpu_device *adev)
2313 {
2314         adev->vcn.ring_jpeg.funcs = &vcn_v1_0_jpeg_ring_vm_funcs;
2315         DRM_INFO("VCN jpeg decode is enabled in VM mode\n");
2316 }
2317
2318 static const struct amdgpu_irq_src_funcs vcn_v1_0_irq_funcs = {
2319         .set = vcn_v1_0_set_interrupt_state,
2320         .process = vcn_v1_0_process_interrupt,
2321 };
2322
2323 static void vcn_v1_0_set_irq_funcs(struct amdgpu_device *adev)
2324 {
2325         adev->vcn.irq.num_types = adev->vcn.num_enc_rings + 2;
2326         adev->vcn.irq.funcs = &vcn_v1_0_irq_funcs;
2327 }
2328
2329 const struct amdgpu_ip_block_version vcn_v1_0_ip_block =
2330 {
2331                 .type = AMD_IP_BLOCK_TYPE_VCN,
2332                 .major = 1,
2333                 .minor = 0,
2334                 .rev = 0,
2335                 .funcs = &vcn_v1_0_ip_funcs,
2336 };