drivers/gpu/drm/amd/amdgpu/ci_dpm.c

   1 /*
   2  * Copyright 2013 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  */
  23
  24 #include <linux/firmware.h>
  25 #include "drmP.h"
  26 #include "amdgpu.h"
  27 #include "amdgpu_pm.h"
  28 #include "amdgpu_ucode.h"
  29 #include "cikd.h"
  30 #include "amdgpu_dpm.h"
  31 #include "ci_dpm.h"
  32 #include "gfx_v7_0.h"
  33 #include "atom.h"
  34 #include <linux/seq_file.h>
  35
  36 #include "smu/smu_7_0_1_d.h"
  37 #include "smu/smu_7_0_1_sh_mask.h"
  38
  39 #include "dce/dce_8_0_d.h"
  40 #include "dce/dce_8_0_sh_mask.h"
  41
  42 #include "bif/bif_4_1_d.h"
  43 #include "bif/bif_4_1_sh_mask.h"
  44
  45 #include "gca/gfx_7_2_d.h"
  46 #include "gca/gfx_7_2_sh_mask.h"
  47
  48 #include "gmc/gmc_7_1_d.h"
  49 #include "gmc/gmc_7_1_sh_mask.h"
  50
  51 /*(DEBLOBBED)*/
  52
  53 #define MC_CG_ARB_FREQ_F0           0x0a
  54 #define MC_CG_ARB_FREQ_F1           0x0b
  55 #define MC_CG_ARB_FREQ_F2           0x0c
  56 #define MC_CG_ARB_FREQ_F3           0x0d
  57
  58 #define SMC_RAM_END 0x40000
  59
  60 #define VOLTAGE_SCALE               4
  61 #define VOLTAGE_VID_OFFSET_SCALE1    625
  62 #define VOLTAGE_VID_OFFSET_SCALE2    100
  63
  64 static const struct ci_pt_defaults defaults_hawaii_xt =
  65 {
  66         1, 0xF, 0xFD, 0x19, 5, 0x14, 0, 0xB0000,
  67         { 0x2E,  0x00,  0x00,  0x88,  0x00,  0x00,  0x72,  0x60,  0x51,  0xA7,  0x79,  0x6B,  0x90,  0xBD,  0x79  },
  68         { 0x217, 0x217, 0x217, 0x242, 0x242, 0x242, 0x269, 0x269, 0x269, 0x2A1, 0x2A1, 0x2A1, 0x2C9, 0x2C9, 0x2C9 }
  69 };
  70
  71 static const struct ci_pt_defaults defaults_hawaii_pro =
  72 {
  73         1, 0xF, 0xFD, 0x19, 5, 0x14, 0, 0x65062,
  74         { 0x2E,  0x00,  0x00,  0x88,  0x00,  0x00,  0x72,  0x60,  0x51,  0xA7,  0x79,  0x6B,  0x90,  0xBD,  0x79  },
  75         { 0x217, 0x217, 0x217, 0x242, 0x242, 0x242, 0x269, 0x269, 0x269, 0x2A1, 0x2A1, 0x2A1, 0x2C9, 0x2C9, 0x2C9 }
  76 };
  77
  78 static const struct ci_pt_defaults defaults_bonaire_xt =
  79 {
  80         1, 0xF, 0xFD, 0x19, 5, 45, 0, 0xB0000,
  81         { 0x79,  0x253, 0x25D, 0xAE,  0x72,  0x80,  0x83,  0x86,  0x6F,  0xC8,  0xC9,  0xC9,  0x2F,  0x4D,  0x61  },
  82         { 0x17C, 0x172, 0x180, 0x1BC, 0x1B3, 0x1BD, 0x206, 0x200, 0x203, 0x25D, 0x25A, 0x255, 0x2C3, 0x2C5, 0x2B4 }
  83 };
  84
  85 static const struct ci_pt_defaults defaults_bonaire_pro =
  86 {
  87         1, 0xF, 0xFD, 0x19, 5, 45, 0, 0x65062,
  88         { 0x8C,  0x23F, 0x244, 0xA6,  0x83,  0x85,  0x86,  0x86,  0x83,  0xDB,  0xDB,  0xDA,  0x67,  0x60,  0x5F  },
  89         { 0x187, 0x193, 0x193, 0x1C7, 0x1D1, 0x1D1, 0x210, 0x219, 0x219, 0x266, 0x26C, 0x26C, 0x2C9, 0x2CB, 0x2CB }
  90 };
  91
  92 static const struct ci_pt_defaults defaults_saturn_xt =
  93 {
  94         1, 0xF, 0xFD, 0x19, 5, 55, 0, 0x70000,
  95         { 0x8C,  0x247, 0x249, 0xA6,  0x80,  0x81,  0x8B,  0x89,  0x86,  0xC9,  0xCA,  0xC9,  0x4D,  0x4D,  0x4D  },
  96         { 0x187, 0x187, 0x187, 0x1C7, 0x1C7, 0x1C7, 0x210, 0x210, 0x210, 0x266, 0x266, 0x266, 0x2C9, 0x2C9, 0x2C9 }
  97 };
  98
  99 static const struct ci_pt_defaults defaults_saturn_pro =
 100 {
 101         1, 0xF, 0xFD, 0x19, 5, 55, 0, 0x30000,
 102         { 0x96,  0x21D, 0x23B, 0xA1,  0x85,  0x87,  0x83,  0x84,  0x81,  0xE6,  0xE6,  0xE6,  0x71,  0x6A,  0x6A  },
 103         { 0x193, 0x19E, 0x19E, 0x1D2, 0x1DC, 0x1DC, 0x21A, 0x223, 0x223, 0x26E, 0x27E, 0x274, 0x2CF, 0x2D2, 0x2D2 }
 104 };
 105
 106 static const struct ci_pt_config_reg didt_config_ci[] =
 107 {
 108         { 0x10, 0x000000ff, 0, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
 109         { 0x10, 0x0000ff00, 8, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
 110         { 0x10, 0x00ff0000, 16, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
 111         { 0x10, 0xff000000, 24, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
 112         { 0x11, 0x000000ff, 0, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
 113         { 0x11, 0x0000ff00, 8, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
 114         { 0x11, 0x00ff0000, 16, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
 115         { 0x11, 0xff000000, 24, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
 116         { 0x12, 0x000000ff, 0, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
 117         { 0x12, 0x0000ff00, 8, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
 118         { 0x12, 0x00ff0000, 16, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
 119         { 0x12, 0xff000000, 24, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
 120         { 0x2, 0x00003fff, 0, 0x4, CISLANDS_CONFIGREG_DIDT_IND },
 121         { 0x2, 0x03ff0000, 16, 0x80, CISLANDS_CONFIGREG_DIDT_IND },
 122         { 0x2, 0x78000000, 27, 0x3, CISLANDS_CONFIGREG_DIDT_IND },
 123         { 0x1, 0x0000ffff, 0, 0x3FFF, CISLANDS_CONFIGREG_DIDT_IND },
 124         { 0x1, 0xffff0000, 16, 0x3FFF, CISLANDS_CONFIGREG_DIDT_IND },
 125         { 0x0, 0x00000001, 0, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
 126         { 0x30, 0x000000ff, 0, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
 127         { 0x30, 0x0000ff00, 8, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
 128         { 0x30, 0x00ff0000, 16, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
 129         { 0x30, 0xff000000, 24, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
 130         { 0x31, 0x000000ff, 0, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
 131         { 0x31, 0x0000ff00, 8, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
 132         { 0x31, 0x00ff0000, 16, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
 133         { 0x31, 0xff000000, 24, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
 134         { 0x32, 0x000000ff, 0, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
 135         { 0x32, 0x0000ff00, 8, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
 136         { 0x32, 0x00ff0000, 16, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
 137         { 0x32, 0xff000000, 24, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
 138         { 0x22, 0x00003fff, 0, 0x4, CISLANDS_CONFIGREG_DIDT_IND },
 139         { 0x22, 0x03ff0000, 16, 0x80, CISLANDS_CONFIGREG_DIDT_IND },
 140         { 0x22, 0x78000000, 27, 0x3, CISLANDS_CONFIGREG_DIDT_IND },
 141         { 0x21, 0x0000ffff, 0, 0x3FFF, CISLANDS_CONFIGREG_DIDT_IND },
 142         { 0x21, 0xffff0000, 16, 0x3FFF, CISLANDS_CONFIGREG_DIDT_IND },
 143         { 0x20, 0x00000001, 0, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
 144         { 0x50, 0x000000ff, 0, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
 145         { 0x50, 0x0000ff00, 8, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
 146         { 0x50, 0x00ff0000, 16, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
 147         { 0x50, 0xff000000, 24, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
 148         { 0x51, 0x000000ff, 0, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
 149         { 0x51, 0x0000ff00, 8, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
 150         { 0x51, 0x00ff0000, 16, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
 151         { 0x51, 0xff000000, 24, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
 152         { 0x52, 0x000000ff, 0, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
 153         { 0x52, 0x0000ff00, 8, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
 154         { 0x52, 0x00ff0000, 16, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
 155         { 0x52, 0xff000000, 24, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
 156         { 0x42, 0x00003fff, 0, 0x4, CISLANDS_CONFIGREG_DIDT_IND },
 157         { 0x42, 0x03ff0000, 16, 0x80, CISLANDS_CONFIGREG_DIDT_IND },
 158         { 0x42, 0x78000000, 27, 0x3, CISLANDS_CONFIGREG_DIDT_IND },
 159         { 0x41, 0x0000ffff, 0, 0x3FFF, CISLANDS_CONFIGREG_DIDT_IND },
 160         { 0x41, 0xffff0000, 16, 0x3FFF, CISLANDS_CONFIGREG_DIDT_IND },
 161         { 0x40, 0x00000001, 0, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
 162         { 0x70, 0x000000ff, 0, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
 163         { 0x70, 0x0000ff00, 8, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
 164         { 0x70, 0x00ff0000, 16, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
 165         { 0x70, 0xff000000, 24, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
 166         { 0x71, 0x000000ff, 0, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
 167         { 0x71, 0x0000ff00, 8, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
 168         { 0x71, 0x00ff0000, 16, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
 169         { 0x71, 0xff000000, 24, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
 170         { 0x72, 0x000000ff, 0, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
 171         { 0x72, 0x0000ff00, 8, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
 172         { 0x72, 0x00ff0000, 16, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
 173         { 0x72, 0xff000000, 24, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
 174         { 0x62, 0x00003fff, 0, 0x4, CISLANDS_CONFIGREG_DIDT_IND },
 175         { 0x62, 0x03ff0000, 16, 0x80, CISLANDS_CONFIGREG_DIDT_IND },
 176         { 0x62, 0x78000000, 27, 0x3, CISLANDS_CONFIGREG_DIDT_IND },
 177         { 0x61, 0x0000ffff, 0, 0x3FFF, CISLANDS_CONFIGREG_DIDT_IND },
 178         { 0x61, 0xffff0000, 16, 0x3FFF, CISLANDS_CONFIGREG_DIDT_IND },
 179         { 0x60, 0x00000001, 0, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
 180         { 0xFFFFFFFF }
 181 };
 182
 183 static u8 ci_get_memory_module_index(struct amdgpu_device *adev)
 184 {
 185         return (u8) ((RREG32(mmBIOS_SCRATCH_4) >> 16) & 0xff);
 186 }
 187
 188 #define MC_CG_ARB_FREQ_F0           0x0a
 189 #define MC_CG_ARB_FREQ_F1           0x0b
 190 #define MC_CG_ARB_FREQ_F2           0x0c
 191 #define MC_CG_ARB_FREQ_F3           0x0d
 192
 193 static int ci_copy_and_switch_arb_sets(struct amdgpu_device *adev,
 194                                        u32 arb_freq_src, u32 arb_freq_dest)
 195 {
 196         u32 mc_arb_dram_timing;
 197         u32 mc_arb_dram_timing2;
 198         u32 burst_time;
 199         u32 mc_cg_config;
 200
 201         switch (arb_freq_src) {
 202         case MC_CG_ARB_FREQ_F0:
 203                 mc_arb_dram_timing  = RREG32(mmMC_ARB_DRAM_TIMING);
 204                 mc_arb_dram_timing2 = RREG32(mmMC_ARB_DRAM_TIMING2);
 205                 burst_time = (RREG32(mmMC_ARB_BURST_TIME) & MC_ARB_BURST_TIME__STATE0_MASK) >>
 206                          MC_ARB_BURST_TIME__STATE0__SHIFT;
 207                 break;
 208         case MC_CG_ARB_FREQ_F1:
 209                 mc_arb_dram_timing  = RREG32(mmMC_ARB_DRAM_TIMING_1);
 210                 mc_arb_dram_timing2 = RREG32(mmMC_ARB_DRAM_TIMING2_1);
 211                 burst_time = (RREG32(mmMC_ARB_BURST_TIME) & MC_ARB_BURST_TIME__STATE1_MASK) >>
 212                          MC_ARB_BURST_TIME__STATE1__SHIFT;
 213                 break;
 214         default:
 215                 return -EINVAL;
 216         }
 217
 218         switch (arb_freq_dest) {
 219         case MC_CG_ARB_FREQ_F0:
 220                 WREG32(mmMC_ARB_DRAM_TIMING, mc_arb_dram_timing);
 221                 WREG32(mmMC_ARB_DRAM_TIMING2, mc_arb_dram_timing2);
 222                 WREG32_P(mmMC_ARB_BURST_TIME, (burst_time << MC_ARB_BURST_TIME__STATE0__SHIFT),
 223                         ~MC_ARB_BURST_TIME__STATE0_MASK);
 224                 break;
 225         case MC_CG_ARB_FREQ_F1:
 226                 WREG32(mmMC_ARB_DRAM_TIMING_1, mc_arb_dram_timing);
 227                 WREG32(mmMC_ARB_DRAM_TIMING2_1, mc_arb_dram_timing2);
 228                 WREG32_P(mmMC_ARB_BURST_TIME, (burst_time << MC_ARB_BURST_TIME__STATE1__SHIFT),
 229                         ~MC_ARB_BURST_TIME__STATE1_MASK);
 230                 break;
 231         default:
 232                 return -EINVAL;
 233         }
 234
 235         mc_cg_config = RREG32(mmMC_CG_CONFIG) | 0x0000000F;
 236         WREG32(mmMC_CG_CONFIG, mc_cg_config);
 237         WREG32_P(mmMC_ARB_CG, (arb_freq_dest) << MC_ARB_CG__CG_ARB_REQ__SHIFT,
 238                 ~MC_ARB_CG__CG_ARB_REQ_MASK);
 239
 240         return 0;
 241 }
 242
 243 static u8 ci_get_ddr3_mclk_frequency_ratio(u32 memory_clock)
 244 {
 245         u8 mc_para_index;
 246
 247         if (memory_clock < 10000)
 248                 mc_para_index = 0;
 249         else if (memory_clock >= 80000)
 250                 mc_para_index = 0x0f;
 251         else
 252                 mc_para_index = (u8)((memory_clock - 10000) / 5000 + 1);
 253         return mc_para_index;
 254 }
 255
 256 static u8 ci_get_mclk_frequency_ratio(u32 memory_clock, bool strobe_mode)
 257 {
 258         u8 mc_para_index;
 259
 260         if (strobe_mode) {
 261                 if (memory_clock < 12500)
 262                         mc_para_index = 0x00;
 263                 else if (memory_clock > 47500)
 264                         mc_para_index = 0x0f;
 265                 else
 266                         mc_para_index = (u8)((memory_clock - 10000) / 2500);
 267         } else {
 268                 if (memory_clock < 65000)
 269                         mc_para_index = 0x00;
 270                 else if (memory_clock > 135000)
 271                         mc_para_index = 0x0f;
 272                 else
 273                         mc_para_index = (u8)((memory_clock - 60000) / 5000);
 274         }
 275         return mc_para_index;
 276 }
 277
 278 static void ci_trim_voltage_table_to_fit_state_table(struct amdgpu_device *adev,
 279                                                      u32 max_voltage_steps,
 280                                                      struct atom_voltage_table *voltage_table)
 281 {
 282         unsigned int i, diff;
 283
 284         if (voltage_table->count <= max_voltage_steps)
 285                 return;
 286
 287         diff = voltage_table->count - max_voltage_steps;
 288
 289         for (i = 0; i < max_voltage_steps; i++)
 290                 voltage_table->entries[i] = voltage_table->entries[i + diff];
 291
 292         voltage_table->count = max_voltage_steps;
 293 }
 294
 295 static int ci_get_std_voltage_value_sidd(struct amdgpu_device *adev,
 296                                          struct atom_voltage_table_entry *voltage_table,
 297                                          u16 *std_voltage_hi_sidd, u16 *std_voltage_lo_sidd);
 298 static int ci_set_power_limit(struct amdgpu_device *adev, u32 n);
 299 static int ci_set_overdrive_target_tdp(struct amdgpu_device *adev,
 300                                        u32 target_tdp);
 301 static int ci_update_uvd_dpm(struct amdgpu_device *adev, bool gate);
 302 static void ci_dpm_set_dpm_funcs(struct amdgpu_device *adev);
 303 static void ci_dpm_set_irq_funcs(struct amdgpu_device *adev);
 304
 305 static PPSMC_Result amdgpu_ci_send_msg_to_smc_with_parameter(struct amdgpu_device *adev,
 306                                                              PPSMC_Msg msg, u32 parameter);
 307 static void ci_thermal_start_smc_fan_control(struct amdgpu_device *adev);
 308 static void ci_fan_ctrl_set_default_mode(struct amdgpu_device *adev);
 309
 310 static struct ci_power_info *ci_get_pi(struct amdgpu_device *adev)
 311 {
 312         struct ci_power_info *pi = adev->pm.dpm.priv;
 313
 314         return pi;
 315 }
 316
 317 static struct ci_ps *ci_get_ps(struct amdgpu_ps *rps)
 318 {
 319         struct ci_ps *ps = rps->ps_priv;
 320
 321         return ps;
 322 }
 323
 324 static void ci_initialize_powertune_defaults(struct amdgpu_device *adev)
 325 {
 326         struct ci_power_info *pi = ci_get_pi(adev);
 327
 328         switch (adev->pdev->device) {
 329         case 0x6649:
 330         case 0x6650:
 331         case 0x6651:
 332         case 0x6658:
 333         case 0x665C:
 334         case 0x665D:
 335         default:
 336                 pi->powertune_defaults = &defaults_bonaire_xt;
 337                 break;
 338         case 0x6640:
 339         case 0x6641:
 340         case 0x6646:
 341         case 0x6647:
 342                 pi->powertune_defaults = &defaults_saturn_xt;
 343                 break;
 344         case 0x67B8:
 345         case 0x67B0:
 346                 pi->powertune_defaults = &defaults_hawaii_xt;
 347                 break;
 348         case 0x67BA:
 349         case 0x67B1:
 350                 pi->powertune_defaults = &defaults_hawaii_pro;
 351                 break;
 352         case 0x67A0:
 353         case 0x67A1:
 354         case 0x67A2:
 355         case 0x67A8:
 356         case 0x67A9:
 357         case 0x67AA:
 358         case 0x67B9:
 359         case 0x67BE:
 360                 pi->powertune_defaults = &defaults_bonaire_xt;
 361                 break;
 362         }
 363
 364         pi->dte_tj_offset = 0;
 365
 366         pi->caps_power_containment = true;
 367         pi->caps_cac = false;
 368         pi->caps_sq_ramping = false;
 369         pi->caps_db_ramping = false;
 370         pi->caps_td_ramping = false;
 371         pi->caps_tcp_ramping = false;
 372
 373         if (pi->caps_power_containment) {
 374                 pi->caps_cac = true;
 375                 if (adev->asic_type == CHIP_HAWAII)
 376                         pi->enable_bapm_feature = false;
 377                 else
 378                         pi->enable_bapm_feature = true;
 379                 pi->enable_tdc_limit_feature = true;
 380                 pi->enable_pkg_pwr_tracking_feature = true;
 381         }
 382 }
 383
 384 static u8 ci_convert_to_vid(u16 vddc)
 385 {
 386         return (6200 - (vddc * VOLTAGE_SCALE)) / 25;
 387 }
 388
 389 static int ci_populate_bapm_vddc_vid_sidd(struct amdgpu_device *adev)
 390 {
 391         struct ci_power_info *pi = ci_get_pi(adev);
 392         u8 *hi_vid = pi->smc_powertune_table.BapmVddCVidHiSidd;
 393         u8 *lo_vid = pi->smc_powertune_table.BapmVddCVidLoSidd;
 394         u8 *hi2_vid = pi->smc_powertune_table.BapmVddCVidHiSidd2;
 395         u32 i;
 396
 397         if (adev->pm.dpm.dyn_state.cac_leakage_table.entries == NULL)
 398                 return -EINVAL;
 399         if (adev->pm.dpm.dyn_state.cac_leakage_table.count > 8)
 400                 return -EINVAL;
 401         if (adev->pm.dpm.dyn_state.cac_leakage_table.count !=
 402             adev->pm.dpm.dyn_state.vddc_dependency_on_sclk.count)
 403                 return -EINVAL;
 404
 405         for (i = 0; i < adev->pm.dpm.dyn_state.cac_leakage_table.count; i++) {
 406                 if (adev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_EVV) {
 407                         lo_vid[i] = ci_convert_to_vid(adev->pm.dpm.dyn_state.cac_leakage_table.entries[i].vddc1);
 408                         hi_vid[i] = ci_convert_to_vid(adev->pm.dpm.dyn_state.cac_leakage_table.entries[i].vddc2);
 409                         hi2_vid[i] = ci_convert_to_vid(adev->pm.dpm.dyn_state.cac_leakage_table.entries[i].vddc3);
 410                 } else {
 411                         lo_vid[i] = ci_convert_to_vid(adev->pm.dpm.dyn_state.cac_leakage_table.entries[i].vddc);
 412                         hi_vid[i] = ci_convert_to_vid((u16)adev->pm.dpm.dyn_state.cac_leakage_table.entries[i].leakage);
 413                 }
 414         }
 415         return 0;
 416 }
 417
 418 static int ci_populate_vddc_vid(struct amdgpu_device *adev)
 419 {
 420         struct ci_power_info *pi = ci_get_pi(adev);
 421         u8 *vid = pi->smc_powertune_table.VddCVid;
 422         u32 i;
 423
 424         if (pi->vddc_voltage_table.count > 8)
 425                 return -EINVAL;
 426
 427         for (i = 0; i < pi->vddc_voltage_table.count; i++)
 428                 vid[i] = ci_convert_to_vid(pi->vddc_voltage_table.entries[i].value);
 429
 430         return 0;
 431 }
 432
 433 static int ci_populate_svi_load_line(struct amdgpu_device *adev)
 434 {
 435         struct ci_power_info *pi = ci_get_pi(adev);
 436         const struct ci_pt_defaults *pt_defaults = pi->powertune_defaults;
 437
 438         pi->smc_powertune_table.SviLoadLineEn = pt_defaults->svi_load_line_en;
 439         pi->smc_powertune_table.SviLoadLineVddC = pt_defaults->svi_load_line_vddc;
 440         pi->smc_powertune_table.SviLoadLineTrimVddC = 3;
 441         pi->smc_powertune_table.SviLoadLineOffsetVddC = 0;
 442
 443         return 0;
 444 }
 445
 446 static int ci_populate_tdc_limit(struct amdgpu_device *adev)
 447 {
 448         struct ci_power_info *pi = ci_get_pi(adev);
 449         const struct ci_pt_defaults *pt_defaults = pi->powertune_defaults;
 450         u16 tdc_limit;
 451
 452         tdc_limit = adev->pm.dpm.dyn_state.cac_tdp_table->tdc * 256;
 453         pi->smc_powertune_table.TDC_VDDC_PkgLimit = cpu_to_be16(tdc_limit);
 454         pi->smc_powertune_table.TDC_VDDC_ThrottleReleaseLimitPerc =
 455                 pt_defaults->tdc_vddc_throttle_release_limit_perc;
 456         pi->smc_powertune_table.TDC_MAWt = pt_defaults->tdc_mawt;
 457
 458         return 0;
 459 }
 460
 461 static int ci_populate_dw8(struct amdgpu_device *adev)
 462 {
 463         struct ci_power_info *pi = ci_get_pi(adev);
 464         const struct ci_pt_defaults *pt_defaults = pi->powertune_defaults;
 465         int ret;
 466
 467         ret = amdgpu_ci_read_smc_sram_dword(adev,
 468                                      SMU7_FIRMWARE_HEADER_LOCATION +
 469                                      offsetof(SMU7_Firmware_Header, PmFuseTable) +
 470                                      offsetof(SMU7_Discrete_PmFuses, TdcWaterfallCtl),
 471                                      (u32 *)&pi->smc_powertune_table.TdcWaterfallCtl,
 472                                      pi->sram_end);
 473         if (ret)
 474                 return -EINVAL;
 475         else
 476                 pi->smc_powertune_table.TdcWaterfallCtl = pt_defaults->tdc_waterfall_ctl;
 477
 478         return 0;
 479 }
 480
 481 static int ci_populate_fuzzy_fan(struct amdgpu_device *adev)
 482 {
 483         struct ci_power_info *pi = ci_get_pi(adev);
 484
 485         if ((adev->pm.dpm.fan.fan_output_sensitivity & (1 << 15)) ||
 486             (adev->pm.dpm.fan.fan_output_sensitivity == 0))
 487                 adev->pm.dpm.fan.fan_output_sensitivity =
 488                         adev->pm.dpm.fan.default_fan_output_sensitivity;
 489
 490         pi->smc_powertune_table.FuzzyFan_PwmSetDelta =
 491                 cpu_to_be16(adev->pm.dpm.fan.fan_output_sensitivity);
 492
 493         return 0;
 494 }
 495
 496 static int ci_min_max_v_gnbl_pm_lid_from_bapm_vddc(struct amdgpu_device *adev)
 497 {
 498         struct ci_power_info *pi = ci_get_pi(adev);
 499         u8 *hi_vid = pi->smc_powertune_table.BapmVddCVidHiSidd;
 500         u8 *lo_vid = pi->smc_powertune_table.BapmVddCVidLoSidd;
 501         int i, min, max;
 502
 503         min = max = hi_vid[0];
 504         for (i = 0; i < 8; i++) {
 505                 if (0 != hi_vid[i]) {
 506                         if (min > hi_vid[i])
 507                                 min = hi_vid[i];
 508                         if (max < hi_vid[i])
 509                                 max = hi_vid[i];
 510                 }
 511
 512                 if (0 != lo_vid[i]) {
 513                         if (min > lo_vid[i])
 514                                 min = lo_vid[i];
 515                         if (max < lo_vid[i])
 516                                 max = lo_vid[i];
 517                 }
 518         }
 519
 520         if ((min == 0) || (max == 0))
 521                 return -EINVAL;
 522         pi->smc_powertune_table.GnbLPMLMaxVid = (u8)max;
 523         pi->smc_powertune_table.GnbLPMLMinVid = (u8)min;
 524
 525         return 0;
 526 }
 527
 528 static int ci_populate_bapm_vddc_base_leakage_sidd(struct amdgpu_device *adev)
 529 {
 530         struct ci_power_info *pi = ci_get_pi(adev);
 531         u16 hi_sidd = pi->smc_powertune_table.BapmVddCBaseLeakageHiSidd;
 532         u16 lo_sidd = pi->smc_powertune_table.BapmVddCBaseLeakageLoSidd;
 533         struct amdgpu_cac_tdp_table *cac_tdp_table =
 534                 adev->pm.dpm.dyn_state.cac_tdp_table;
 535
 536         hi_sidd = cac_tdp_table->high_cac_leakage / 100 * 256;
 537         lo_sidd = cac_tdp_table->low_cac_leakage / 100 * 256;
 538
 539         pi->smc_powertune_table.BapmVddCBaseLeakageHiSidd = cpu_to_be16(hi_sidd);
 540         pi->smc_powertune_table.BapmVddCBaseLeakageLoSidd = cpu_to_be16(lo_sidd);
 541
 542         return 0;
 543 }
 544
 545 static int ci_populate_bapm_parameters_in_dpm_table(struct amdgpu_device *adev)
 546 {
 547         struct ci_power_info *pi = ci_get_pi(adev);
 548         const struct ci_pt_defaults *pt_defaults = pi->powertune_defaults;
 549         SMU7_Discrete_DpmTable  *dpm_table = &pi->smc_state_table;
 550         struct amdgpu_cac_tdp_table *cac_tdp_table =
 551                 adev->pm.dpm.dyn_state.cac_tdp_table;
 552         struct amdgpu_ppm_table *ppm = adev->pm.dpm.dyn_state.ppm_table;
 553         int i, j, k;
 554         const u16 *def1;
 555         const u16 *def2;
 556
 557         dpm_table->DefaultTdp = cac_tdp_table->tdp * 256;
 558         dpm_table->TargetTdp = cac_tdp_table->configurable_tdp * 256;
 559
 560         dpm_table->DTETjOffset = (u8)pi->dte_tj_offset;
 561         dpm_table->GpuTjMax =
 562                 (u8)(pi->thermal_temp_setting.temperature_high / 1000);
 563         dpm_table->GpuTjHyst = 8;
 564
 565         dpm_table->DTEAmbientTempBase = pt_defaults->dte_ambient_temp_base;
 566
 567         if (ppm) {
 568                 dpm_table->PPM_PkgPwrLimit = cpu_to_be16((u16)ppm->dgpu_tdp * 256 / 1000);
 569                 dpm_table->PPM_TemperatureLimit = cpu_to_be16((u16)ppm->tj_max * 256);
 570         } else {
 571                 dpm_table->PPM_PkgPwrLimit = cpu_to_be16(0);
 572                 dpm_table->PPM_TemperatureLimit = cpu_to_be16(0);
 573         }
 574
 575         dpm_table->BAPM_TEMP_GRADIENT = cpu_to_be32(pt_defaults->bapm_temp_gradient);
 576         def1 = pt_defaults->bapmti_r;
 577         def2 = pt_defaults->bapmti_rc;
 578
 579         for (i = 0; i < SMU7_DTE_ITERATIONS; i++) {
 580                 for (j = 0; j < SMU7_DTE_SOURCES; j++) {
 581                         for (k = 0; k < SMU7_DTE_SINKS; k++) {
 582                                 dpm_table->BAPMTI_R[i][j][k] = cpu_to_be16(*def1);
 583                                 dpm_table->BAPMTI_RC[i][j][k] = cpu_to_be16(*def2);
 584                                 def1++;
 585                                 def2++;
 586                         }
 587                 }
 588         }
 589
 590         return 0;
 591 }
 592
 593 static int ci_populate_pm_base(struct amdgpu_device *adev)
 594 {
 595         struct ci_power_info *pi = ci_get_pi(adev);
 596         u32 pm_fuse_table_offset;
 597         int ret;
 598
 599         if (pi->caps_power_containment) {
 600                 ret = amdgpu_ci_read_smc_sram_dword(adev,
 601                                              SMU7_FIRMWARE_HEADER_LOCATION +
 602                                              offsetof(SMU7_Firmware_Header, PmFuseTable),
 603                                              &pm_fuse_table_offset, pi->sram_end);
 604                 if (ret)
 605                         return ret;
 606                 ret = ci_populate_bapm_vddc_vid_sidd(adev);
 607                 if (ret)
 608                         return ret;
 609                 ret = ci_populate_vddc_vid(adev);
 610                 if (ret)
 611                         return ret;
 612                 ret = ci_populate_svi_load_line(adev);
 613                 if (ret)
 614                         return ret;
 615                 ret = ci_populate_tdc_limit(adev);
 616                 if (ret)
 617                         return ret;
 618                 ret = ci_populate_dw8(adev);
 619                 if (ret)
 620                         return ret;
 621                 ret = ci_populate_fuzzy_fan(adev);
 622                 if (ret)
 623                         return ret;
 624                 ret = ci_min_max_v_gnbl_pm_lid_from_bapm_vddc(adev);
 625                 if (ret)
 626                         return ret;
 627                 ret = ci_populate_bapm_vddc_base_leakage_sidd(adev);
 628                 if (ret)
 629                         return ret;
 630                 ret = amdgpu_ci_copy_bytes_to_smc(adev, pm_fuse_table_offset,
 631                                            (u8 *)&pi->smc_powertune_table,
 632                                            sizeof(SMU7_Discrete_PmFuses), pi->sram_end);
 633                 if (ret)
 634                         return ret;
 635         }
 636
 637         return 0;
 638 }
 639
 640 static void ci_do_enable_didt(struct amdgpu_device *adev, const bool enable)
 641 {
 642         struct ci_power_info *pi = ci_get_pi(adev);
 643         u32 data;
 644
 645         if (pi->caps_sq_ramping) {
 646                 data = RREG32_DIDT(ixDIDT_SQ_CTRL0);
 647                 if (enable)
 648                         data |= DIDT_SQ_CTRL0__DIDT_CTRL_EN_MASK;
 649                 else
 650                         data &= ~DIDT_SQ_CTRL0__DIDT_CTRL_EN_MASK;
 651                 WREG32_DIDT(ixDIDT_SQ_CTRL0, data);
 652         }
 653
 654         if (pi->caps_db_ramping) {
 655                 data = RREG32_DIDT(ixDIDT_DB_CTRL0);
 656                 if (enable)
 657                         data |= DIDT_DB_CTRL0__DIDT_CTRL_EN_MASK;
 658                 else
 659                         data &= ~DIDT_DB_CTRL0__DIDT_CTRL_EN_MASK;
 660                 WREG32_DIDT(ixDIDT_DB_CTRL0, data);
 661         }
 662
 663         if (pi->caps_td_ramping) {
 664                 data = RREG32_DIDT(ixDIDT_TD_CTRL0);
 665                 if (enable)
 666                         data |= DIDT_TD_CTRL0__DIDT_CTRL_EN_MASK;
 667                 else
 668                         data &= ~DIDT_TD_CTRL0__DIDT_CTRL_EN_MASK;
 669                 WREG32_DIDT(ixDIDT_TD_CTRL0, data);
 670         }
 671
 672         if (pi->caps_tcp_ramping) {
 673                 data = RREG32_DIDT(ixDIDT_TCP_CTRL0);
 674                 if (enable)
 675                         data |= DIDT_TCP_CTRL0__DIDT_CTRL_EN_MASK;
 676                 else
 677                         data &= ~DIDT_TCP_CTRL0__DIDT_CTRL_EN_MASK;
 678                 WREG32_DIDT(ixDIDT_TCP_CTRL0, data);
 679         }
 680 }
 681
 682 static int ci_program_pt_config_registers(struct amdgpu_device *adev,
 683                                           const struct ci_pt_config_reg *cac_config_regs)
 684 {
 685         const struct ci_pt_config_reg *config_regs = cac_config_regs;
 686         u32 data;
 687         u32 cache = 0;
 688
 689         if (config_regs == NULL)
 690                 return -EINVAL;
 691
 692         while (config_regs->offset != 0xFFFFFFFF) {
 693                 if (config_regs->type == CISLANDS_CONFIGREG_CACHE) {
 694                         cache |= ((config_regs->value << config_regs->shift) & config_regs->mask);
 695                 } else {
 696                         switch (config_regs->type) {
 697                         case CISLANDS_CONFIGREG_SMC_IND:
 698                                 data = RREG32_SMC(config_regs->offset);
 699                                 break;
 700                         case CISLANDS_CONFIGREG_DIDT_IND:
 701                                 data = RREG32_DIDT(config_regs->offset);
 702                                 break;
 703                         default:
 704                                 data = RREG32(config_regs->offset);
 705                                 break;
 706                         }
 707
 708                         data &= ~config_regs->mask;
 709                         data |= ((config_regs->value << config_regs->shift) & config_regs->mask);
 710                         data |= cache;
 711
 712                         switch (config_regs->type) {
 713                         case CISLANDS_CONFIGREG_SMC_IND:
 714                                 WREG32_SMC(config_regs->offset, data);
 715                                 break;
 716                         case CISLANDS_CONFIGREG_DIDT_IND:
 717                                 WREG32_DIDT(config_regs->offset, data);
 718                                 break;
 719                         default:
 720                                 WREG32(config_regs->offset, data);
 721                                 break;
 722                         }
 723                         cache = 0;
 724                 }
 725                 config_regs++;
 726         }
 727         return 0;
 728 }
 729
 730 static int ci_enable_didt(struct amdgpu_device *adev, bool enable)
 731 {
 732         struct ci_power_info *pi = ci_get_pi(adev);
 733         int ret;
 734
 735         if (pi->caps_sq_ramping || pi->caps_db_ramping ||
 736             pi->caps_td_ramping || pi->caps_tcp_ramping) {
 737                 gfx_v7_0_enter_rlc_safe_mode(adev);
 738
 739                 if (enable) {
 740                         ret = ci_program_pt_config_registers(adev, didt_config_ci);
 741                         if (ret) {
 742                                 gfx_v7_0_exit_rlc_safe_mode(adev);
 743                                 return ret;
 744                         }
 745                 }
 746
 747                 ci_do_enable_didt(adev, enable);
 748
 749                 gfx_v7_0_exit_rlc_safe_mode(adev);
 750         }
 751
 752         return 0;
 753 }
 754
 755 static int ci_enable_power_containment(struct amdgpu_device *adev, bool enable)
 756 {
 757         struct ci_power_info *pi = ci_get_pi(adev);
 758         PPSMC_Result smc_result;
 759         int ret = 0;
 760
 761         if (enable) {
 762                 pi->power_containment_features = 0;
 763                 if (pi->caps_power_containment) {
 764                         if (pi->enable_bapm_feature) {
 765                                 smc_result = amdgpu_ci_send_msg_to_smc(adev, PPSMC_MSG_EnableDTE);
 766                                 if (smc_result != PPSMC_Result_OK)
 767                                         ret = -EINVAL;
 768                                 else
 769                                         pi->power_containment_features |= POWERCONTAINMENT_FEATURE_BAPM;
 770                         }
 771
 772                         if (pi->enable_tdc_limit_feature) {
 773                                 smc_result = amdgpu_ci_send_msg_to_smc(adev, PPSMC_MSG_TDCLimitEnable);
 774                                 if (smc_result != PPSMC_Result_OK)
 775                                         ret = -EINVAL;
 776                                 else
 777                                         pi->power_containment_features |= POWERCONTAINMENT_FEATURE_TDCLimit;
 778                         }
 779
 780                         if (pi->enable_pkg_pwr_tracking_feature) {
 781                                 smc_result = amdgpu_ci_send_msg_to_smc(adev, PPSMC_MSG_PkgPwrLimitEnable);
 782                                 if (smc_result != PPSMC_Result_OK) {
 783                                         ret = -EINVAL;
 784                                 } else {
 785                                         struct amdgpu_cac_tdp_table *cac_tdp_table =
 786                                                 adev->pm.dpm.dyn_state.cac_tdp_table;
 787                                         u32 default_pwr_limit =
 788                                                 (u32)(cac_tdp_table->maximum_power_delivery_limit * 256);
 789
 790                                         pi->power_containment_features |= POWERCONTAINMENT_FEATURE_PkgPwrLimit;
 791
 792                                         ci_set_power_limit(adev, default_pwr_limit);
 793                                 }
 794                         }
 795                 }
 796         } else {
 797                 if (pi->caps_power_containment && pi->power_containment_features) {
 798                         if (pi->power_containment_features & POWERCONTAINMENT_FEATURE_TDCLimit)
 799                                 amdgpu_ci_send_msg_to_smc(adev, PPSMC_MSG_TDCLimitDisable);
 800
 801                         if (pi->power_containment_features & POWERCONTAINMENT_FEATURE_BAPM)
 802                                 amdgpu_ci_send_msg_to_smc(adev, PPSMC_MSG_DisableDTE);
 803
 804                         if (pi->power_containment_features & POWERCONTAINMENT_FEATURE_PkgPwrLimit)
 805                                 amdgpu_ci_send_msg_to_smc(adev, PPSMC_MSG_PkgPwrLimitDisable);
 806                         pi->power_containment_features = 0;
 807                 }
 808         }
 809
 810         return ret;
 811 }
 812
 813 static int ci_enable_smc_cac(struct amdgpu_device *adev, bool enable)
 814 {
 815         struct ci_power_info *pi = ci_get_pi(adev);
 816         PPSMC_Result smc_result;
 817         int ret = 0;
 818
 819         if (pi->caps_cac) {
 820                 if (enable) {
 821                         smc_result = amdgpu_ci_send_msg_to_smc(adev, PPSMC_MSG_EnableCac);
 822                         if (smc_result != PPSMC_Result_OK) {
 823                                 ret = -EINVAL;
 824                                 pi->cac_enabled = false;
 825                         } else {
 826                                 pi->cac_enabled = true;
 827                         }
 828                 } else if (pi->cac_enabled) {
 829                         amdgpu_ci_send_msg_to_smc(adev, PPSMC_MSG_DisableCac);
 830                         pi->cac_enabled = false;
 831                 }
 832         }
 833
 834         return ret;
 835 }
 836
 837 static int ci_enable_thermal_based_sclk_dpm(struct amdgpu_device *adev,
 838                                             bool enable)
 839 {
 840         struct ci_power_info *pi = ci_get_pi(adev);
 841         PPSMC_Result smc_result = PPSMC_Result_OK;
 842
 843         if (pi->thermal_sclk_dpm_enabled) {
 844                 if (enable)
 845                         smc_result = amdgpu_ci_send_msg_to_smc(adev, PPSMC_MSG_ENABLE_THERMAL_DPM);
 846                 else
 847                         smc_result = amdgpu_ci_send_msg_to_smc(adev, PPSMC_MSG_DISABLE_THERMAL_DPM);
 848         }
 849
 850         if (smc_result == PPSMC_Result_OK)
 851                 return 0;
 852         else
 853                 return -EINVAL;
 854 }
 855
 856 static int ci_power_control_set_level(struct amdgpu_device *adev)
 857 {
 858         struct ci_power_info *pi = ci_get_pi(adev);
 859         struct amdgpu_cac_tdp_table *cac_tdp_table =
 860                 adev->pm.dpm.dyn_state.cac_tdp_table;
 861         s32 adjust_percent;
 862         s32 target_tdp;
 863         int ret = 0;
 864         bool adjust_polarity = false; /* ??? */
 865
 866         if (pi->caps_power_containment) {
 867                 adjust_percent = adjust_polarity ?
 868                         adev->pm.dpm.tdp_adjustment : (-1 * adev->pm.dpm.tdp_adjustment);
 869                 target_tdp = ((100 + adjust_percent) *
 870                               (s32)cac_tdp_table->configurable_tdp) / 100;
 871
 872                 ret = ci_set_overdrive_target_tdp(adev, (u32)target_tdp);
 873         }
 874
 875         return ret;
 876 }
 877
 878 static void ci_dpm_powergate_uvd(struct amdgpu_device *adev, bool gate)
 879 {
 880         struct ci_power_info *pi = ci_get_pi(adev);
 881
 882         if (pi->uvd_power_gated == gate)
 883                 return;
 884
 885         pi->uvd_power_gated = gate;
 886
 887         ci_update_uvd_dpm(adev, gate);
 888 }
 889
 890 static bool ci_dpm_vblank_too_short(struct amdgpu_device *adev)
 891 {
 892         u32 vblank_time = amdgpu_dpm_get_vblank_time(adev);
 893         u32 switch_limit = adev->mc.vram_type == AMDGPU_VRAM_TYPE_GDDR5 ? 450 : 300;
 894
 895         /* disable mclk switching if the refresh is >120Hz, even if the
 896          * blanking period would allow it
 897          */
 898         if (amdgpu_dpm_get_vrefresh(adev) > 120)
 899                 return true;
 900
 901         if (vblank_time < switch_limit)
 902                 return true;
 903         else
 904                 return false;
 905
 906 }
 907
 908 static void ci_apply_state_adjust_rules(struct amdgpu_device *adev,
 909                                         struct amdgpu_ps *rps)
 910 {
 911         struct ci_ps *ps = ci_get_ps(rps);
 912         struct ci_power_info *pi = ci_get_pi(adev);
 913         struct amdgpu_clock_and_voltage_limits *max_limits;
 914         bool disable_mclk_switching;
 915         u32 sclk, mclk;
 916         int i;
 917
 918         if (rps->vce_active) {
 919                 rps->evclk = adev->pm.dpm.vce_states[adev->pm.dpm.vce_level].evclk;
 920                 rps->ecclk = adev->pm.dpm.vce_states[adev->pm.dpm.vce_level].ecclk;
 921         } else {
 922                 rps->evclk = 0;
 923                 rps->ecclk = 0;
 924         }
 925
 926         if ((adev->pm.dpm.new_active_crtc_count > 1) ||
 927             ci_dpm_vblank_too_short(adev))
 928                 disable_mclk_switching = true;
 929         else
 930                 disable_mclk_switching = false;
 931
 932         if ((rps->class & ATOM_PPLIB_CLASSIFICATION_UI_MASK) == ATOM_PPLIB_CLASSIFICATION_UI_BATTERY)
 933                 pi->battery_state = true;
 934         else
 935                 pi->battery_state = false;
 936
 937         if (adev->pm.dpm.ac_power)
 938                 max_limits = &adev->pm.dpm.dyn_state.max_clock_voltage_on_ac;
 939         else
 940                 max_limits = &adev->pm.dpm.dyn_state.max_clock_voltage_on_dc;
 941
 942         if (adev->pm.dpm.ac_power == false) {
 943                 for (i = 0; i < ps->performance_level_count; i++) {
 944                         if (ps->performance_levels[i].mclk > max_limits->mclk)
 945                                 ps->performance_levels[i].mclk = max_limits->mclk;
 946                         if (ps->performance_levels[i].sclk > max_limits->sclk)
 947                                 ps->performance_levels[i].sclk = max_limits->sclk;
 948                 }
 949         }
 950
 951         /* XXX validate the min clocks required for display */
 952
 953         if (disable_mclk_switching) {
 954                 mclk  = ps->performance_levels[ps->performance_level_count - 1].mclk;
 955                 sclk = ps->performance_levels[0].sclk;
 956         } else {
 957                 mclk = ps->performance_levels[0].mclk;
 958                 sclk = ps->performance_levels[0].sclk;
 959         }
 960
 961         if (rps->vce_active) {
 962                 if (sclk < adev->pm.dpm.vce_states[adev->pm.dpm.vce_level].sclk)
 963                         sclk = adev->pm.dpm.vce_states[adev->pm.dpm.vce_level].sclk;
 964                 if (mclk < adev->pm.dpm.vce_states[adev->pm.dpm.vce_level].mclk)
 965                         mclk = adev->pm.dpm.vce_states[adev->pm.dpm.vce_level].mclk;
 966         }
 967
 968         ps->performance_levels[0].sclk = sclk;
 969         ps->performance_levels[0].mclk = mclk;
 970
 971         if (ps->performance_levels[1].sclk < ps->performance_levels[0].sclk)
 972                 ps->performance_levels[1].sclk = ps->performance_levels[0].sclk;
 973
 974         if (disable_mclk_switching) {
 975                 if (ps->performance_levels[0].mclk < ps->performance_levels[1].mclk)
 976                         ps->performance_levels[0].mclk = ps->performance_levels[1].mclk;
 977         } else {
 978                 if (ps->performance_levels[1].mclk < ps->performance_levels[0].mclk)
 979                         ps->performance_levels[1].mclk = ps->performance_levels[0].mclk;
 980         }
 981 }
 982
 983 static int ci_thermal_set_temperature_range(struct amdgpu_device *adev,
 984                                             int min_temp, int max_temp)
 985 {
 986         int low_temp = 0 * 1000;
 987         int high_temp = 255 * 1000;
 988         u32 tmp;
 989
 990         if (low_temp < min_temp)
 991                 low_temp = min_temp;
 992         if (high_temp > max_temp)
 993                 high_temp = max_temp;
 994         if (high_temp < low_temp) {
 995                 DRM_ERROR("invalid thermal range: %d - %d\n", low_temp, high_temp);
 996                 return -EINVAL;
 997         }
 998
 999         tmp = RREG32_SMC(ixCG_THERMAL_INT);
1000         tmp &= ~(CG_THERMAL_INT__DIG_THERM_INTH_MASK | CG_THERMAL_INT__DIG_THERM_INTL_MASK);
1001         tmp |= ((high_temp / 1000) << CG_THERMAL_INT__DIG_THERM_INTH__SHIFT) |
1002                 ((low_temp / 1000)) << CG_THERMAL_INT__DIG_THERM_INTL__SHIFT;
1003         WREG32_SMC(ixCG_THERMAL_INT, tmp);
1004
1005 #if 0
1006         /* XXX: need to figure out how to handle this properly */
1007         tmp = RREG32_SMC(ixCG_THERMAL_CTRL);
1008         tmp &= DIG_THERM_DPM_MASK;
1009         tmp |= DIG_THERM_DPM(high_temp / 1000);
1010         WREG32_SMC(ixCG_THERMAL_CTRL, tmp);
1011 #endif
1012
1013         adev->pm.dpm.thermal.min_temp = low_temp;
1014         adev->pm.dpm.thermal.max_temp = high_temp;
1015         return 0;
1016 }
1017
1018 static int ci_thermal_enable_alert(struct amdgpu_device *adev,
1019                                    bool enable)
1020 {
1021         u32 thermal_int = RREG32_SMC(ixCG_THERMAL_INT);
1022         PPSMC_Result result;
1023
1024         if (enable) {
1025                 thermal_int &= ~(CG_THERMAL_INT_CTRL__THERM_INTH_MASK_MASK |
1026                                  CG_THERMAL_INT_CTRL__THERM_INTL_MASK_MASK);
1027                 WREG32_SMC(ixCG_THERMAL_INT, thermal_int);
1028                 result = amdgpu_ci_send_msg_to_smc(adev, PPSMC_MSG_Thermal_Cntl_Enable);
1029                 if (result != PPSMC_Result_OK) {
1030                         DRM_DEBUG_KMS("Could not enable thermal interrupts.\n");
1031                         return -EINVAL;
1032                 }
1033         } else {
1034                 thermal_int |= CG_THERMAL_INT_CTRL__THERM_INTH_MASK_MASK |
1035                         CG_THERMAL_INT_CTRL__THERM_INTL_MASK_MASK;
1036                 WREG32_SMC(ixCG_THERMAL_INT, thermal_int);
1037                 result = amdgpu_ci_send_msg_to_smc(adev, PPSMC_MSG_Thermal_Cntl_Disable);
1038                 if (result != PPSMC_Result_OK) {
1039                         DRM_DEBUG_KMS("Could not disable thermal interrupts.\n");
1040                         return -EINVAL;
1041                 }
1042         }
1043
1044         return 0;
1045 }
1046
1047 static void ci_fan_ctrl_set_static_mode(struct amdgpu_device *adev, u32 mode)
1048 {
1049         struct ci_power_info *pi = ci_get_pi(adev);
1050         u32 tmp;
1051
1052         if (pi->fan_ctrl_is_in_default_mode) {
1053                 tmp = (RREG32_SMC(ixCG_FDO_CTRL2) & CG_FDO_CTRL2__FDO_PWM_MODE_MASK)
1054                         >> CG_FDO_CTRL2__FDO_PWM_MODE__SHIFT;
1055                 pi->fan_ctrl_default_mode = tmp;
1056                 tmp = (RREG32_SMC(ixCG_FDO_CTRL2) & CG_FDO_CTRL2__TMIN_MASK)
1057                         >> CG_FDO_CTRL2__TMIN__SHIFT;
1058                 pi->t_min = tmp;
1059                 pi->fan_ctrl_is_in_default_mode = false;
1060         }
1061
1062         tmp = RREG32_SMC(ixCG_FDO_CTRL2) & ~CG_FDO_CTRL2__TMIN_MASK;
1063         tmp |= 0 << CG_FDO_CTRL2__TMIN__SHIFT;
1064         WREG32_SMC(ixCG_FDO_CTRL2, tmp);
1065
1066         tmp = RREG32_SMC(ixCG_FDO_CTRL2) & ~CG_FDO_CTRL2__FDO_PWM_MODE_MASK;
1067         tmp |= mode << CG_FDO_CTRL2__FDO_PWM_MODE__SHIFT;
1068         WREG32_SMC(ixCG_FDO_CTRL2, tmp);
1069 }
1070
1071 static int ci_thermal_setup_fan_table(struct amdgpu_device *adev)
1072 {
1073         struct ci_power_info *pi = ci_get_pi(adev);
1074         SMU7_Discrete_FanTable fan_table = { FDO_MODE_HARDWARE };
1075         u32 duty100;
1076         u32 t_diff1, t_diff2, pwm_diff1, pwm_diff2;
1077         u16 fdo_min, slope1, slope2;
1078         u32 reference_clock, tmp;
1079         int ret;
1080         u64 tmp64;
1081
1082         if (!pi->fan_table_start) {
1083                 adev->pm.dpm.fan.ucode_fan_control = false;
1084                 return 0;
1085         }
1086
1087         duty100 = (RREG32_SMC(ixCG_FDO_CTRL1) & CG_FDO_CTRL1__FMAX_DUTY100_MASK)
1088                 >> CG_FDO_CTRL1__FMAX_DUTY100__SHIFT;
1089
1090         if (duty100 == 0) {
1091                 adev->pm.dpm.fan.ucode_fan_control = false;
1092                 return 0;
1093         }
1094
1095         tmp64 = (u64)adev->pm.dpm.fan.pwm_min * duty100;
1096         do_div(tmp64, 10000);
1097         fdo_min = (u16)tmp64;
1098
1099         t_diff1 = adev->pm.dpm.fan.t_med - adev->pm.dpm.fan.t_min;
1100         t_diff2 = adev->pm.dpm.fan.t_high - adev->pm.dpm.fan.t_med;
1101
1102         pwm_diff1 = adev->pm.dpm.fan.pwm_med - adev->pm.dpm.fan.pwm_min;
1103         pwm_diff2 = adev->pm.dpm.fan.pwm_high - adev->pm.dpm.fan.pwm_med;
1104
1105         slope1 = (u16)((50 + ((16 * duty100 * pwm_diff1) / t_diff1)) / 100);
1106         slope2 = (u16)((50 + ((16 * duty100 * pwm_diff2) / t_diff2)) / 100);
1107
1108         fan_table.TempMin = cpu_to_be16((50 + adev->pm.dpm.fan.t_min) / 100);
1109         fan_table.TempMed = cpu_to_be16((50 + adev->pm.dpm.fan.t_med) / 100);
1110         fan_table.TempMax = cpu_to_be16((50 + adev->pm.dpm.fan.t_max) / 100);
1111
1112         fan_table.Slope1 = cpu_to_be16(slope1);
1113         fan_table.Slope2 = cpu_to_be16(slope2);
1114
1115         fan_table.FdoMin = cpu_to_be16(fdo_min);
1116
1117         fan_table.HystDown = cpu_to_be16(adev->pm.dpm.fan.t_hyst);
1118
1119         fan_table.HystUp = cpu_to_be16(1);
1120
1121         fan_table.HystSlope = cpu_to_be16(1);
1122
1123         fan_table.TempRespLim = cpu_to_be16(5);
1124
1125         reference_clock = amdgpu_asic_get_xclk(adev);
1126
1127         fan_table.RefreshPeriod = cpu_to_be32((adev->pm.dpm.fan.cycle_delay *
1128                                                reference_clock) / 1600);
1129
1130         fan_table.FdoMax = cpu_to_be16((u16)duty100);
1131
1132         tmp = (RREG32_SMC(ixCG_MULT_THERMAL_CTRL) & CG_MULT_THERMAL_CTRL__TEMP_SEL_MASK)
1133                 >> CG_MULT_THERMAL_CTRL__TEMP_SEL__SHIFT;
1134         fan_table.TempSrc = (uint8_t)tmp;
1135
1136         ret = amdgpu_ci_copy_bytes_to_smc(adev,
1137                                           pi->fan_table_start,
1138                                           (u8 *)(&fan_table),
1139                                           sizeof(fan_table),
1140                                           pi->sram_end);
1141
1142         if (ret) {
1143                 DRM_ERROR("Failed to load fan table to the SMC.");
1144                 adev->pm.dpm.fan.ucode_fan_control = false;
1145         }
1146
1147         return 0;
1148 }
1149
1150 static int ci_fan_ctrl_start_smc_fan_control(struct amdgpu_device *adev)
1151 {
1152         struct ci_power_info *pi = ci_get_pi(adev);
1153         PPSMC_Result ret;
1154
1155         if (pi->caps_od_fuzzy_fan_control_support) {
1156                 ret = amdgpu_ci_send_msg_to_smc_with_parameter(adev,
1157                                                                PPSMC_StartFanControl,
1158                                                                FAN_CONTROL_FUZZY);
1159                 if (ret != PPSMC_Result_OK)
1160                         return -EINVAL;
1161                 ret = amdgpu_ci_send_msg_to_smc_with_parameter(adev,
1162                                                                PPSMC_MSG_SetFanPwmMax,
1163                                                                adev->pm.dpm.fan.default_max_fan_pwm);
1164                 if (ret != PPSMC_Result_OK)
1165                         return -EINVAL;
1166         } else {
1167                 ret = amdgpu_ci_send_msg_to_smc_with_parameter(adev,
1168                                                                PPSMC_StartFanControl,
1169                                                                FAN_CONTROL_TABLE);
1170                 if (ret != PPSMC_Result_OK)
1171                         return -EINVAL;
1172         }
1173
1174         pi->fan_is_controlled_by_smc = true;
1175         return 0;
1176 }
1177
1178
1179 static int ci_fan_ctrl_stop_smc_fan_control(struct amdgpu_device *adev)
1180 {
1181         PPSMC_Result ret;
1182         struct ci_power_info *pi = ci_get_pi(adev);
1183
1184         ret = amdgpu_ci_send_msg_to_smc(adev, PPSMC_StopFanControl);
1185         if (ret == PPSMC_Result_OK) {
1186                 pi->fan_is_controlled_by_smc = false;
1187                 return 0;
1188         } else {
1189                 return -EINVAL;
1190         }
1191 }
1192
1193 static int ci_dpm_get_fan_speed_percent(struct amdgpu_device *adev,
1194                                         u32 *speed)
1195 {
1196         u32 duty, duty100;
1197         u64 tmp64;
1198
1199         if (adev->pm.no_fan)
1200                 return -ENOENT;
1201
1202         duty100 = (RREG32_SMC(ixCG_FDO_CTRL1) & CG_FDO_CTRL1__FMAX_DUTY100_MASK)
1203                 >> CG_FDO_CTRL1__FMAX_DUTY100__SHIFT;
1204         duty = (RREG32_SMC(ixCG_THERMAL_STATUS) & CG_THERMAL_STATUS__FDO_PWM_DUTY_MASK)
1205                 >> CG_THERMAL_STATUS__FDO_PWM_DUTY__SHIFT;
1206
1207         if (duty100 == 0)
1208                 return -EINVAL;
1209
1210         tmp64 = (u64)duty * 100;
1211         do_div(tmp64, duty100);
1212         *speed = (u32)tmp64;
1213
1214         if (*speed > 100)
1215                 *speed = 100;
1216
1217         return 0;
1218 }
1219
1220 static int ci_dpm_set_fan_speed_percent(struct amdgpu_device *adev,
1221                                         u32 speed)
1222 {
1223         u32 tmp;
1224         u32 duty, duty100;
1225         u64 tmp64;
1226         struct ci_power_info *pi = ci_get_pi(adev);
1227
1228         if (adev->pm.no_fan)
1229                 return -ENOENT;
1230
1231         if (pi->fan_is_controlled_by_smc)
1232                 return -EINVAL;
1233
1234         if (speed > 100)
1235                 return -EINVAL;
1236
1237         duty100 = (RREG32_SMC(ixCG_FDO_CTRL1) & CG_FDO_CTRL1__FMAX_DUTY100_MASK)
1238                 >> CG_FDO_CTRL1__FMAX_DUTY100__SHIFT;
1239
1240         if (duty100 == 0)
1241                 return -EINVAL;
1242
1243         tmp64 = (u64)speed * duty100;
1244         do_div(tmp64, 100);
1245         duty = (u32)tmp64;
1246
1247         tmp = RREG32_SMC(ixCG_FDO_CTRL0) & ~CG_FDO_CTRL0__FDO_STATIC_DUTY_MASK;
1248         tmp |= duty << CG_FDO_CTRL0__FDO_STATIC_DUTY__SHIFT;
1249         WREG32_SMC(ixCG_FDO_CTRL0, tmp);
1250
1251         return 0;
1252 }
1253
1254 static void ci_dpm_set_fan_control_mode(struct amdgpu_device *adev, u32 mode)
1255 {
1256         if (mode) {
1257                 /* stop auto-manage */
1258                 if (adev->pm.dpm.fan.ucode_fan_control)
1259                         ci_fan_ctrl_stop_smc_fan_control(adev);
1260                 ci_fan_ctrl_set_static_mode(adev, mode);
1261         } else {
1262                 /* restart auto-manage */
1263                 if (adev->pm.dpm.fan.ucode_fan_control)
1264                         ci_thermal_start_smc_fan_control(adev);
1265                 else
1266                         ci_fan_ctrl_set_default_mode(adev);
1267         }
1268 }
1269
1270 static u32 ci_dpm_get_fan_control_mode(struct amdgpu_device *adev)
1271 {
1272         struct ci_power_info *pi = ci_get_pi(adev);
1273         u32 tmp;
1274
1275         if (pi->fan_is_controlled_by_smc)
1276                 return 0;
1277
1278         tmp = RREG32_SMC(ixCG_FDO_CTRL2) & CG_FDO_CTRL2__FDO_PWM_MODE_MASK;
1279         return (tmp >> CG_FDO_CTRL2__FDO_PWM_MODE__SHIFT);
1280 }
1281
1282 #if 0
1283 static int ci_fan_ctrl_get_fan_speed_rpm(struct amdgpu_device *adev,
1284                                          u32 *speed)
1285 {
1286         u32 tach_period;
1287         u32 xclk = amdgpu_asic_get_xclk(adev);
1288
1289         if (adev->pm.no_fan)
1290                 return -ENOENT;
1291
1292         if (adev->pm.fan_pulses_per_revolution == 0)
1293                 return -ENOENT;
1294
1295         tach_period = (RREG32_SMC(ixCG_TACH_STATUS) & CG_TACH_STATUS__TACH_PERIOD_MASK)
1296                 >> CG_TACH_STATUS__TACH_PERIOD__SHIFT;
1297         if (tach_period == 0)
1298                 return -ENOENT;
1299
1300         *speed = 60 * xclk * 10000 / tach_period;
1301
1302         return 0;
1303 }
1304
1305 static int ci_fan_ctrl_set_fan_speed_rpm(struct amdgpu_device *adev,
1306                                          u32 speed)
1307 {
1308         u32 tach_period, tmp;
1309         u32 xclk = amdgpu_asic_get_xclk(adev);
1310
1311         if (adev->pm.no_fan)
1312                 return -ENOENT;
1313
1314         if (adev->pm.fan_pulses_per_revolution == 0)
1315                 return -ENOENT;
1316
1317         if ((speed < adev->pm.fan_min_rpm) ||
1318             (speed > adev->pm.fan_max_rpm))
1319                 return -EINVAL;
1320
1321         if (adev->pm.dpm.fan.ucode_fan_control)
1322                 ci_fan_ctrl_stop_smc_fan_control(adev);
1323
1324         tach_period = 60 * xclk * 10000 / (8 * speed);
1325         tmp = RREG32_SMC(ixCG_TACH_CTRL) & ~CG_TACH_CTRL__TARGET_PERIOD_MASK;
1326         tmp |= tach_period << CG_TACH_CTRL__TARGET_PERIOD__SHIFT;
1327         WREG32_SMC(CG_TACH_CTRL, tmp);
1328
1329         ci_fan_ctrl_set_static_mode(adev, FDO_PWM_MODE_STATIC_RPM);
1330
1331         return 0;
1332 }
1333 #endif
1334
1335 static void ci_fan_ctrl_set_default_mode(struct amdgpu_device *adev)
1336 {
1337         struct ci_power_info *pi = ci_get_pi(adev);
1338         u32 tmp;
1339
1340         if (!pi->fan_ctrl_is_in_default_mode) {
1341                 tmp = RREG32_SMC(ixCG_FDO_CTRL2) & ~CG_FDO_CTRL2__FDO_PWM_MODE_MASK;
1342                 tmp |= pi->fan_ctrl_default_mode << CG_FDO_CTRL2__FDO_PWM_MODE__SHIFT;
1343                 WREG32_SMC(ixCG_FDO_CTRL2, tmp);
1344
1345                 tmp = RREG32_SMC(ixCG_FDO_CTRL2) & ~CG_FDO_CTRL2__TMIN_MASK;
1346                 tmp |= pi->t_min << CG_FDO_CTRL2__TMIN__SHIFT;
1347                 WREG32_SMC(ixCG_FDO_CTRL2, tmp);
1348                 pi->fan_ctrl_is_in_default_mode = true;
1349         }
1350 }
1351
1352 static void ci_thermal_start_smc_fan_control(struct amdgpu_device *adev)
1353 {
1354         if (adev->pm.dpm.fan.ucode_fan_control) {
1355                 ci_fan_ctrl_start_smc_fan_control(adev);
1356                 ci_fan_ctrl_set_static_mode(adev, FDO_PWM_MODE_STATIC);
1357         }
1358 }
1359
1360 static void ci_thermal_initialize(struct amdgpu_device *adev)
1361 {
1362         u32 tmp;
1363
1364         if (adev->pm.fan_pulses_per_revolution) {
1365                 tmp = RREG32_SMC(ixCG_TACH_CTRL) & ~CG_TACH_CTRL__EDGE_PER_REV_MASK;
1366                 tmp |= (adev->pm.fan_pulses_per_revolution - 1)
1367                         << CG_TACH_CTRL__EDGE_PER_REV__SHIFT;
1368                 WREG32_SMC(ixCG_TACH_CTRL, tmp);
1369         }
1370
1371         tmp = RREG32_SMC(ixCG_FDO_CTRL2) & ~CG_FDO_CTRL2__TACH_PWM_RESP_RATE_MASK;
1372         tmp |= 0x28 << CG_FDO_CTRL2__TACH_PWM_RESP_RATE__SHIFT;
1373         WREG32_SMC(ixCG_FDO_CTRL2, tmp);
1374 }
1375
1376 static int ci_thermal_start_thermal_controller(struct amdgpu_device *adev)
1377 {
1378         int ret;
1379
1380         ci_thermal_initialize(adev);
1381         ret = ci_thermal_set_temperature_range(adev, CISLANDS_TEMP_RANGE_MIN, CISLANDS_TEMP_RANGE_MAX);
1382         if (ret)
1383                 return ret;
1384         ret = ci_thermal_enable_alert(adev, true);
1385         if (ret)
1386                 return ret;
1387         if (adev->pm.dpm.fan.ucode_fan_control) {
1388                 ret = ci_thermal_setup_fan_table(adev);
1389                 if (ret)
1390                         return ret;
1391                 ci_thermal_start_smc_fan_control(adev);
1392         }
1393
1394         return 0;
1395 }
1396
1397 static void ci_thermal_stop_thermal_controller(struct amdgpu_device *adev)
1398 {
1399         if (!adev->pm.no_fan)
1400                 ci_fan_ctrl_set_default_mode(adev);
1401 }
1402
1403 #if 0
1404 static int ci_read_smc_soft_register(struct amdgpu_device *adev,
1405                                      u16 reg_offset, u32 *value)
1406 {
1407         struct ci_power_info *pi = ci_get_pi(adev);
1408
1409         return amdgpu_ci_read_smc_sram_dword(adev,
1410                                       pi->soft_regs_start + reg_offset,
1411                                       value, pi->sram_end);
1412 }
1413 #endif
1414
1415 static int ci_write_smc_soft_register(struct amdgpu_device *adev,
1416                                       u16 reg_offset, u32 value)
1417 {
1418         struct ci_power_info *pi = ci_get_pi(adev);
1419
1420         return amdgpu_ci_write_smc_sram_dword(adev,
1421                                        pi->soft_regs_start + reg_offset,
1422                                        value, pi->sram_end);
1423 }
1424
1425 static void ci_init_fps_limits(struct amdgpu_device *adev)
1426 {
1427         struct ci_power_info *pi = ci_get_pi(adev);
1428         SMU7_Discrete_DpmTable *table = &pi->smc_state_table;
1429
1430         if (pi->caps_fps) {
1431                 u16 tmp;
1432
1433                 tmp = 45;
1434                 table->FpsHighT = cpu_to_be16(tmp);
1435
1436                 tmp = 30;
1437                 table->FpsLowT = cpu_to_be16(tmp);
1438         }
1439 }
1440
1441 static int ci_update_sclk_t(struct amdgpu_device *adev)
1442 {
1443         struct ci_power_info *pi = ci_get_pi(adev);
1444         int ret = 0;
1445         u32 low_sclk_interrupt_t = 0;
1446
1447         if (pi->caps_sclk_throttle_low_notification) {
1448                 low_sclk_interrupt_t = cpu_to_be32(pi->low_sclk_interrupt_t);
1449
1450                 ret = amdgpu_ci_copy_bytes_to_smc(adev,
1451                                            pi->dpm_table_start +
1452                                            offsetof(SMU7_Discrete_DpmTable, LowSclkInterruptT),
1453                                            (u8 *)&low_sclk_interrupt_t,
1454                                            sizeof(u32), pi->sram_end);
1455
1456         }
1457
1458         return ret;
1459 }
1460
1461 static void ci_get_leakage_voltages(struct amdgpu_device *adev)
1462 {
1463         struct ci_power_info *pi = ci_get_pi(adev);
1464         u16 leakage_id, virtual_voltage_id;
1465         u16 vddc, vddci;
1466         int i;
1467
1468         pi->vddc_leakage.count = 0;
1469         pi->vddci_leakage.count = 0;
1470
1471         if (adev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_EVV) {
1472                 for (i = 0; i < CISLANDS_MAX_LEAKAGE_COUNT; i++) {
1473                         virtual_voltage_id = ATOM_VIRTUAL_VOLTAGE_ID0 + i;
1474                         if (amdgpu_atombios_get_voltage_evv(adev, virtual_voltage_id, &vddc) != 0)
1475                                 continue;
1476                         if (vddc != 0 && vddc != virtual_voltage_id) {
1477                                 pi->vddc_leakage.actual_voltage[pi->vddc_leakage.count] = vddc;
1478                                 pi->vddc_leakage.leakage_id[pi->vddc_leakage.count] = virtual_voltage_id;
1479                                 pi->vddc_leakage.count++;
1480                         }
1481                 }
1482         } else if (amdgpu_atombios_get_leakage_id_from_vbios(adev, &leakage_id) == 0) {
1483                 for (i = 0; i < CISLANDS_MAX_LEAKAGE_COUNT; i++) {
1484                         virtual_voltage_id = ATOM_VIRTUAL_VOLTAGE_ID0 + i;
1485                         if (amdgpu_atombios_get_leakage_vddc_based_on_leakage_params(adev, &vddc, &vddci,
1486                                                                                      virtual_voltage_id,
1487                                                                                      leakage_id) == 0) {
1488                                 if (vddc != 0 && vddc != virtual_voltage_id) {
1489                                         pi->vddc_leakage.actual_voltage[pi->vddc_leakage.count] = vddc;
1490                                         pi->vddc_leakage.leakage_id[pi->vddc_leakage.count] = virtual_voltage_id;
1491                                         pi->vddc_leakage.count++;
1492                                 }
1493                                 if (vddci != 0 && vddci != virtual_voltage_id) {
1494                                         pi->vddci_leakage.actual_voltage[pi->vddci_leakage.count] = vddci;
1495                                         pi->vddci_leakage.leakage_id[pi->vddci_leakage.count] = virtual_voltage_id;
1496                                         pi->vddci_leakage.count++;
1497                                 }
1498                         }
1499                 }
1500         }
1501 }
1502
1503 static void ci_set_dpm_event_sources(struct amdgpu_device *adev, u32 sources)
1504 {
1505         struct ci_power_info *pi = ci_get_pi(adev);
1506         bool want_thermal_protection;
1507         enum amdgpu_dpm_event_src dpm_event_src;
1508         u32 tmp;
1509
1510         switch (sources) {
1511         case 0:
1512         default:
1513                 want_thermal_protection = false;
1514                 break;
1515         case (1 << AMDGPU_DPM_AUTO_THROTTLE_SRC_THERMAL):
1516                 want_thermal_protection = true;
1517                 dpm_event_src = AMDGPU_DPM_EVENT_SRC_DIGITAL;
1518                 break;
1519         case (1 << AMDGPU_DPM_AUTO_THROTTLE_SRC_EXTERNAL):
1520                 want_thermal_protection = true;
1521                 dpm_event_src = AMDGPU_DPM_EVENT_SRC_EXTERNAL;
1522                 break;
1523         case ((1 << AMDGPU_DPM_AUTO_THROTTLE_SRC_EXTERNAL) |
1524               (1 << AMDGPU_DPM_AUTO_THROTTLE_SRC_THERMAL)):
1525                 want_thermal_protection = true;
1526                 dpm_event_src = AMDGPU_DPM_EVENT_SRC_DIGIAL_OR_EXTERNAL;
1527                 break;
1528         }
1529
1530         if (want_thermal_protection) {
1531 #if 0
1532                 /* XXX: need to figure out how to handle this properly */
1533                 tmp = RREG32_SMC(ixCG_THERMAL_CTRL);
1534                 tmp &= DPM_EVENT_SRC_MASK;
1535                 tmp |= DPM_EVENT_SRC(dpm_event_src);
1536                 WREG32_SMC(ixCG_THERMAL_CTRL, tmp);
1537 #endif
1538
1539                 tmp = RREG32_SMC(ixGENERAL_PWRMGT);
1540                 if (pi->thermal_protection)
1541                         tmp &= ~GENERAL_PWRMGT__THERMAL_PROTECTION_DIS_MASK;
1542                 else
1543                         tmp |= GENERAL_PWRMGT__THERMAL_PROTECTION_DIS_MASK;
1544                 WREG32_SMC(ixGENERAL_PWRMGT, tmp);
1545         } else {
1546                 tmp = RREG32_SMC(ixGENERAL_PWRMGT);
1547                 tmp |= GENERAL_PWRMGT__THERMAL_PROTECTION_DIS_MASK;
1548                 WREG32_SMC(ixGENERAL_PWRMGT, tmp);
1549         }
1550 }
1551
1552 static void ci_enable_auto_throttle_source(struct amdgpu_device *adev,
1553                                            enum amdgpu_dpm_auto_throttle_src source,
1554                                            bool enable)
1555 {
1556         struct ci_power_info *pi = ci_get_pi(adev);
1557
1558         if (enable) {
1559                 if (!(pi->active_auto_throttle_sources & (1 << source))) {
1560                         pi->active_auto_throttle_sources |= 1 << source;
1561                         ci_set_dpm_event_sources(adev, pi->active_auto_throttle_sources);
1562                 }
1563         } else {
1564                 if (pi->active_auto_throttle_sources & (1 << source)) {
1565                         pi->active_auto_throttle_sources &= ~(1 << source);
1566                         ci_set_dpm_event_sources(adev, pi->active_auto_throttle_sources);
1567                 }
1568         }
1569 }
1570
1571 static void ci_enable_vr_hot_gpio_interrupt(struct amdgpu_device *adev)
1572 {
1573         if (adev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_REGULATOR_HOT)
1574                 amdgpu_ci_send_msg_to_smc(adev, PPSMC_MSG_EnableVRHotGPIOInterrupt);
1575 }
1576
1577 static int ci_unfreeze_sclk_mclk_dpm(struct amdgpu_device *adev)
1578 {
1579         struct ci_power_info *pi = ci_get_pi(adev);
1580         PPSMC_Result smc_result;
1581
1582         if (!pi->need_update_smu7_dpm_table)
1583                 return 0;
1584
1585         if ((!pi->sclk_dpm_key_disabled) &&
1586             (pi->need_update_smu7_dpm_table & (DPMTABLE_OD_UPDATE_SCLK | DPMTABLE_UPDATE_SCLK))) {
1587                 smc_result = amdgpu_ci_send_msg_to_smc(adev, PPSMC_MSG_SCLKDPM_UnfreezeLevel);
1588                 if (smc_result != PPSMC_Result_OK)
1589                         return -EINVAL;
1590         }
1591
1592         if ((!pi->mclk_dpm_key_disabled) &&
1593             (pi->need_update_smu7_dpm_table & DPMTABLE_OD_UPDATE_MCLK)) {
1594                 smc_result = amdgpu_ci_send_msg_to_smc(adev, PPSMC_MSG_MCLKDPM_UnfreezeLevel);
1595                 if (smc_result != PPSMC_Result_OK)
1596                         return -EINVAL;
1597         }
1598
1599         pi->need_update_smu7_dpm_table = 0;
1600         return 0;
1601 }
1602
1603 static int ci_enable_sclk_mclk_dpm(struct amdgpu_device *adev, bool enable)
1604 {
1605         struct ci_power_info *pi = ci_get_pi(adev);
1606         PPSMC_Result smc_result;
1607
1608         if (enable) {
1609                 if (!pi->sclk_dpm_key_disabled) {
1610                         smc_result = amdgpu_ci_send_msg_to_smc(adev, PPSMC_MSG_DPM_Enable);
1611                         if (smc_result != PPSMC_Result_OK)
1612                                 return -EINVAL;
1613                 }
1614
1615                 if (!pi->mclk_dpm_key_disabled) {
1616                         smc_result = amdgpu_ci_send_msg_to_smc(adev, PPSMC_MSG_MCLKDPM_Enable);
1617                         if (smc_result != PPSMC_Result_OK)
1618                                 return -EINVAL;
1619
1620                         WREG32_P(mmMC_SEQ_CNTL_3, MC_SEQ_CNTL_3__CAC_EN_MASK,
1621                                         ~MC_SEQ_CNTL_3__CAC_EN_MASK);
1622
1623                         WREG32_SMC(ixLCAC_MC0_CNTL, 0x05);
1624                         WREG32_SMC(ixLCAC_MC1_CNTL, 0x05);
1625                         WREG32_SMC(ixLCAC_CPL_CNTL, 0x100005);
1626
1627                         udelay(10);
1628
1629                         WREG32_SMC(ixLCAC_MC0_CNTL, 0x400005);
1630                         WREG32_SMC(ixLCAC_MC1_CNTL, 0x400005);
1631                         WREG32_SMC(ixLCAC_CPL_CNTL, 0x500005);
1632                 }
1633         } else {
1634                 if (!pi->sclk_dpm_key_disabled) {
1635                         smc_result = amdgpu_ci_send_msg_to_smc(adev, PPSMC_MSG_DPM_Disable);
1636                         if (smc_result != PPSMC_Result_OK)
1637                                 return -EINVAL;
1638                 }
1639
1640                 if (!pi->mclk_dpm_key_disabled) {
1641                         smc_result = amdgpu_ci_send_msg_to_smc(adev, PPSMC_MSG_MCLKDPM_Disable);
1642                         if (smc_result != PPSMC_Result_OK)
1643                                 return -EINVAL;
1644                 }
1645         }
1646
1647         return 0;
1648 }
1649
1650 static int ci_start_dpm(struct amdgpu_device *adev)
1651 {
1652         struct ci_power_info *pi = ci_get_pi(adev);
1653         PPSMC_Result smc_result;
1654         int ret;
1655         u32 tmp;
1656
1657         tmp = RREG32_SMC(ixGENERAL_PWRMGT);
1658         tmp |= GENERAL_PWRMGT__GLOBAL_PWRMGT_EN_MASK;
1659         WREG32_SMC(ixGENERAL_PWRMGT, tmp);
1660
1661         tmp = RREG32_SMC(ixSCLK_PWRMGT_CNTL);
1662         tmp |= SCLK_PWRMGT_CNTL__DYNAMIC_PM_EN_MASK;
1663         WREG32_SMC(ixSCLK_PWRMGT_CNTL, tmp);
1664
1665         ci_write_smc_soft_register(adev, offsetof(SMU7_SoftRegisters, VoltageChangeTimeout), 0x1000);
1666
1667         WREG32_P(mmBIF_LNCNT_RESET, 0, ~BIF_LNCNT_RESET__RESET_LNCNT_EN_MASK);
1668
1669         smc_result = amdgpu_ci_send_msg_to_smc(adev, PPSMC_MSG_Voltage_Cntl_Enable);
1670         if (smc_result != PPSMC_Result_OK)
1671                 return -EINVAL;
1672
1673         ret = ci_enable_sclk_mclk_dpm(adev, true);
1674         if (ret)
1675                 return ret;
1676
1677         if (!pi->pcie_dpm_key_disabled) {
1678                 smc_result = amdgpu_ci_send_msg_to_smc(adev, PPSMC_MSG_PCIeDPM_Enable);
1679                 if (smc_result != PPSMC_Result_OK)
1680                         return -EINVAL;
1681         }
1682
1683         return 0;
1684 }
1685
1686 static int ci_freeze_sclk_mclk_dpm(struct amdgpu_device *adev)
1687 {
1688         struct ci_power_info *pi = ci_get_pi(adev);
1689         PPSMC_Result smc_result;
1690
1691         if (!pi->need_update_smu7_dpm_table)
1692                 return 0;
1693
1694         if ((!pi->sclk_dpm_key_disabled) &&
1695             (pi->need_update_smu7_dpm_table & (DPMTABLE_OD_UPDATE_SCLK | DPMTABLE_UPDATE_SCLK))) {
1696                 smc_result = amdgpu_ci_send_msg_to_smc(adev, PPSMC_MSG_SCLKDPM_FreezeLevel);
1697                 if (smc_result != PPSMC_Result_OK)
1698                         return -EINVAL;
1699         }
1700
1701         if ((!pi->mclk_dpm_key_disabled) &&
1702             (pi->need_update_smu7_dpm_table & DPMTABLE_OD_UPDATE_MCLK)) {
1703                 smc_result = amdgpu_ci_send_msg_to_smc(adev, PPSMC_MSG_MCLKDPM_FreezeLevel);
1704                 if (smc_result != PPSMC_Result_OK)
1705                         return -EINVAL;
1706         }
1707
1708         return 0;
1709 }
1710
1711 static int ci_stop_dpm(struct amdgpu_device *adev)
1712 {
1713         struct ci_power_info *pi = ci_get_pi(adev);
1714         PPSMC_Result smc_result;
1715         int ret;
1716         u32 tmp;
1717
1718         tmp = RREG32_SMC(ixGENERAL_PWRMGT);
1719         tmp &= ~GENERAL_PWRMGT__GLOBAL_PWRMGT_EN_MASK;
1720         WREG32_SMC(ixGENERAL_PWRMGT, tmp);
1721
1722         tmp = RREG32_SMC(ixSCLK_PWRMGT_CNTL);
1723         tmp &= ~SCLK_PWRMGT_CNTL__DYNAMIC_PM_EN_MASK;
1724         WREG32_SMC(ixSCLK_PWRMGT_CNTL, tmp);
1725
1726         if (!pi->pcie_dpm_key_disabled) {
1727                 smc_result = amdgpu_ci_send_msg_to_smc(adev, PPSMC_MSG_PCIeDPM_Disable);
1728                 if (smc_result != PPSMC_Result_OK)
1729                         return -EINVAL;
1730         }
1731
1732         ret = ci_enable_sclk_mclk_dpm(adev, false);
1733         if (ret)
1734                 return ret;
1735
1736         smc_result = amdgpu_ci_send_msg_to_smc(adev, PPSMC_MSG_Voltage_Cntl_Disable);
1737         if (smc_result != PPSMC_Result_OK)
1738                 return -EINVAL;
1739
1740         return 0;
1741 }
1742
1743 static void ci_enable_sclk_control(struct amdgpu_device *adev, bool enable)
1744 {
1745         u32 tmp = RREG32_SMC(ixSCLK_PWRMGT_CNTL);
1746
1747         if (enable)
1748                 tmp &= ~SCLK_PWRMGT_CNTL__SCLK_PWRMGT_OFF_MASK;
1749         else
1750                 tmp |= SCLK_PWRMGT_CNTL__SCLK_PWRMGT_OFF_MASK;
1751         WREG32_SMC(ixSCLK_PWRMGT_CNTL, tmp);
1752 }
1753
1754 #if 0
1755 static int ci_notify_hw_of_power_source(struct amdgpu_device *adev,
1756                                         bool ac_power)
1757 {
1758         struct ci_power_info *pi = ci_get_pi(adev);
1759         struct amdgpu_cac_tdp_table *cac_tdp_table =
1760                 adev->pm.dpm.dyn_state.cac_tdp_table;
1761         u32 power_limit;
1762
1763         if (ac_power)
1764                 power_limit = (u32)(cac_tdp_table->maximum_power_delivery_limit * 256);
1765         else
1766                 power_limit = (u32)(cac_tdp_table->battery_power_limit * 256);
1767
1768         ci_set_power_limit(adev, power_limit);
1769
1770         if (pi->caps_automatic_dc_transition) {
1771                 if (ac_power)
1772                         amdgpu_ci_send_msg_to_smc(adev, PPSMC_MSG_RunningOnAC);
1773                 else
1774                         amdgpu_ci_send_msg_to_smc(adev, PPSMC_MSG_Remove_DC_Clamp);
1775         }
1776
1777         return 0;
1778 }
1779 #endif
1780
1781 static PPSMC_Result amdgpu_ci_send_msg_to_smc_with_parameter(struct amdgpu_device *adev,
1782                                                       PPSMC_Msg msg, u32 parameter)
1783 {
1784         WREG32(mmSMC_MSG_ARG_0, parameter);
1785         return amdgpu_ci_send_msg_to_smc(adev, msg);
1786 }
1787
1788 static PPSMC_Result amdgpu_ci_send_msg_to_smc_return_parameter(struct amdgpu_device *adev,
1789                                                         PPSMC_Msg msg, u32 *parameter)
1790 {
1791         PPSMC_Result smc_result;
1792
1793         smc_result = amdgpu_ci_send_msg_to_smc(adev, msg);
1794
1795         if ((smc_result == PPSMC_Result_OK) && parameter)
1796                 *parameter = RREG32(mmSMC_MSG_ARG_0);
1797
1798         return smc_result;
1799 }
1800
1801 static int ci_dpm_force_state_sclk(struct amdgpu_device *adev, u32 n)
1802 {
1803         struct ci_power_info *pi = ci_get_pi(adev);
1804
1805         if (!pi->sclk_dpm_key_disabled) {
1806                 PPSMC_Result smc_result =
1807                         amdgpu_ci_send_msg_to_smc_with_parameter(adev, PPSMC_MSG_SCLKDPM_SetEnabledMask, 1 << n);
1808                 if (smc_result != PPSMC_Result_OK)
1809                         return -EINVAL;
1810         }
1811
1812         return 0;
1813 }
1814
1815 static int ci_dpm_force_state_mclk(struct amdgpu_device *adev, u32 n)
1816 {
1817         struct ci_power_info *pi = ci_get_pi(adev);
1818
1819         if (!pi->mclk_dpm_key_disabled) {
1820                 PPSMC_Result smc_result =
1821                         amdgpu_ci_send_msg_to_smc_with_parameter(adev, PPSMC_MSG_MCLKDPM_SetEnabledMask, 1 << n);
1822                 if (smc_result != PPSMC_Result_OK)
1823                         return -EINVAL;
1824         }
1825
1826         return 0;
1827 }
1828
1829 static int ci_dpm_force_state_pcie(struct amdgpu_device *adev, u32 n)
1830 {
1831         struct ci_power_info *pi = ci_get_pi(adev);
1832
1833         if (!pi->pcie_dpm_key_disabled) {
1834                 PPSMC_Result smc_result =
1835                         amdgpu_ci_send_msg_to_smc_with_parameter(adev, PPSMC_MSG_PCIeDPM_ForceLevel, n);
1836                 if (smc_result != PPSMC_Result_OK)
1837                         return -EINVAL;
1838         }
1839
1840         return 0;
1841 }
1842
1843 static int ci_set_power_limit(struct amdgpu_device *adev, u32 n)
1844 {
1845         struct ci_power_info *pi = ci_get_pi(adev);
1846
1847         if (pi->power_containment_features & POWERCONTAINMENT_FEATURE_PkgPwrLimit) {
1848                 PPSMC_Result smc_result =
1849                         amdgpu_ci_send_msg_to_smc_with_parameter(adev, PPSMC_MSG_PkgPwrSetLimit, n);
1850                 if (smc_result != PPSMC_Result_OK)
1851                         return -EINVAL;
1852         }
1853
1854         return 0;
1855 }
1856
1857 static int ci_set_overdrive_target_tdp(struct amdgpu_device *adev,
1858                                        u32 target_tdp)
1859 {
1860         PPSMC_Result smc_result =
1861                 amdgpu_ci_send_msg_to_smc_with_parameter(adev, PPSMC_MSG_OverDriveSetTargetTdp, target_tdp);
1862         if (smc_result != PPSMC_Result_OK)
1863                 return -EINVAL;
1864         return 0;
1865 }
1866
1867 #if 0
1868 static int ci_set_boot_state(struct amdgpu_device *adev)
1869 {
1870         return ci_enable_sclk_mclk_dpm(adev, false);
1871 }
1872 #endif
1873
1874 static u32 ci_get_average_sclk_freq(struct amdgpu_device *adev)
1875 {
1876         u32 sclk_freq;
1877         PPSMC_Result smc_result =
1878                 amdgpu_ci_send_msg_to_smc_return_parameter(adev,
1879                                                     PPSMC_MSG_API_GetSclkFrequency,
1880                                                     &sclk_freq);
1881         if (smc_result != PPSMC_Result_OK)
1882                 sclk_freq = 0;
1883
1884         return sclk_freq;
1885 }
1886
1887 static u32 ci_get_average_mclk_freq(struct amdgpu_device *adev)
1888 {
1889         u32 mclk_freq;
1890         PPSMC_Result smc_result =
1891                 amdgpu_ci_send_msg_to_smc_return_parameter(adev,
1892                                                     PPSMC_MSG_API_GetMclkFrequency,
1893                                                     &mclk_freq);
1894         if (smc_result != PPSMC_Result_OK)
1895                 mclk_freq = 0;
1896
1897         return mclk_freq;
1898 }
1899
1900 static void ci_dpm_start_smc(struct amdgpu_device *adev)
1901 {
1902         int i;
1903
1904         amdgpu_ci_program_jump_on_start(adev);
1905         amdgpu_ci_start_smc_clock(adev);
1906         amdgpu_ci_start_smc(adev);
1907         for (i = 0; i < adev->usec_timeout; i++) {
1908                 if (RREG32_SMC(ixFIRMWARE_FLAGS) & FIRMWARE_FLAGS__INTERRUPTS_ENABLED_MASK)
1909                         break;
1910         }
1911 }
1912
1913 static void ci_dpm_stop_smc(struct amdgpu_device *adev)
1914 {
1915         amdgpu_ci_reset_smc(adev);
1916         amdgpu_ci_stop_smc_clock(adev);
1917 }
1918
1919 static int ci_process_firmware_header(struct amdgpu_device *adev)
1920 {
1921         struct ci_power_info *pi = ci_get_pi(adev);
1922         u32 tmp;
1923         int ret;
1924
1925         ret = amdgpu_ci_read_smc_sram_dword(adev,
1926                                      SMU7_FIRMWARE_HEADER_LOCATION +
1927                                      offsetof(SMU7_Firmware_Header, DpmTable),
1928                                      &tmp, pi->sram_end);
1929         if (ret)
1930                 return ret;
1931
1932         pi->dpm_table_start = tmp;
1933
1934         ret = amdgpu_ci_read_smc_sram_dword(adev,
1935                                      SMU7_FIRMWARE_HEADER_LOCATION +
1936                                      offsetof(SMU7_Firmware_Header, SoftRegisters),
1937                                      &tmp, pi->sram_end);
1938         if (ret)
1939                 return ret;
1940
1941         pi->soft_regs_start = tmp;
1942
1943         ret = amdgpu_ci_read_smc_sram_dword(adev,
1944                                      SMU7_FIRMWARE_HEADER_LOCATION +
1945                                      offsetof(SMU7_Firmware_Header, mcRegisterTable),
1946                                      &tmp, pi->sram_end);
1947         if (ret)
1948                 return ret;
1949
1950         pi->mc_reg_table_start = tmp;
1951
1952         ret = amdgpu_ci_read_smc_sram_dword(adev,
1953                                      SMU7_FIRMWARE_HEADER_LOCATION +
1954                                      offsetof(SMU7_Firmware_Header, FanTable),
1955                                      &tmp, pi->sram_end);
1956         if (ret)
1957                 return ret;
1958
1959         pi->fan_table_start = tmp;
1960
1961         ret = amdgpu_ci_read_smc_sram_dword(adev,
1962                                      SMU7_FIRMWARE_HEADER_LOCATION +
1963                                      offsetof(SMU7_Firmware_Header, mcArbDramTimingTable),
1964                                      &tmp, pi->sram_end);
1965         if (ret)
1966                 return ret;
1967
1968         pi->arb_table_start = tmp;
1969
1970         return 0;
1971 }
1972
1973 static void ci_read_clock_registers(struct amdgpu_device *adev)
1974 {
1975         struct ci_power_info *pi = ci_get_pi(adev);
1976
1977         pi->clock_registers.cg_spll_func_cntl =
1978                 RREG32_SMC(ixCG_SPLL_FUNC_CNTL);
1979         pi->clock_registers.cg_spll_func_cntl_2 =
1980                 RREG32_SMC(ixCG_SPLL_FUNC_CNTL_2);
1981         pi->clock_registers.cg_spll_func_cntl_3 =
1982                 RREG32_SMC(ixCG_SPLL_FUNC_CNTL_3);
1983         pi->clock_registers.cg_spll_func_cntl_4 =
1984                 RREG32_SMC(ixCG_SPLL_FUNC_CNTL_4);
1985         pi->clock_registers.cg_spll_spread_spectrum =
1986                 RREG32_SMC(ixCG_SPLL_SPREAD_SPECTRUM);
1987         pi->clock_registers.cg_spll_spread_spectrum_2 =
1988                 RREG32_SMC(ixCG_SPLL_SPREAD_SPECTRUM_2);
1989         pi->clock_registers.dll_cntl = RREG32(mmDLL_CNTL);
1990         pi->clock_registers.mclk_pwrmgt_cntl = RREG32(mmMCLK_PWRMGT_CNTL);
1991         pi->clock_registers.mpll_ad_func_cntl = RREG32(mmMPLL_AD_FUNC_CNTL);
1992         pi->clock_registers.mpll_dq_func_cntl = RREG32(mmMPLL_DQ_FUNC_CNTL);
1993         pi->clock_registers.mpll_func_cntl = RREG32(mmMPLL_FUNC_CNTL);
1994         pi->clock_registers.mpll_func_cntl_1 = RREG32(mmMPLL_FUNC_CNTL_1);
1995         pi->clock_registers.mpll_func_cntl_2 = RREG32(mmMPLL_FUNC_CNTL_2);
1996         pi->clock_registers.mpll_ss1 = RREG32(mmMPLL_SS1);
1997         pi->clock_registers.mpll_ss2 = RREG32(mmMPLL_SS2);
1998 }
1999
2000 static void ci_init_sclk_t(struct amdgpu_device *adev)
2001 {
2002         struct ci_power_info *pi = ci_get_pi(adev);
2003
2004         pi->low_sclk_interrupt_t = 0;
2005 }
2006
2007 static void ci_enable_thermal_protection(struct amdgpu_device *adev,
2008                                          bool enable)
2009 {
2010         u32 tmp = RREG32_SMC(ixGENERAL_PWRMGT);
2011
2012         if (enable)
2013                 tmp &= ~GENERAL_PWRMGT__THERMAL_PROTECTION_DIS_MASK;
2014         else
2015                 tmp |= GENERAL_PWRMGT__THERMAL_PROTECTION_DIS_MASK;
2016         WREG32_SMC(ixGENERAL_PWRMGT, tmp);
2017 }
2018
2019 static void ci_enable_acpi_power_management(struct amdgpu_device *adev)
2020 {
2021         u32 tmp = RREG32_SMC(ixGENERAL_PWRMGT);
2022
2023         tmp |= GENERAL_PWRMGT__STATIC_PM_EN_MASK;
2024
2025         WREG32_SMC(ixGENERAL_PWRMGT, tmp);
2026 }
2027
2028 #if 0
2029 static int ci_enter_ulp_state(struct amdgpu_device *adev)
2030 {
2031
2032         WREG32(mmSMC_MESSAGE_0, PPSMC_MSG_SwitchToMinimumPower);
2033
2034         udelay(25000);
2035
2036         return 0;
2037 }
2038
2039 static int ci_exit_ulp_state(struct amdgpu_device *adev)
2040 {
2041         int i;
2042
2043         WREG32(mmSMC_MESSAGE_0, PPSMC_MSG_ResumeFromMinimumPower);
2044
2045         udelay(7000);
2046
2047         for (i = 0; i < adev->usec_timeout; i++) {
2048                 if (RREG32(mmSMC_RESP_0) == 1)
2049                         break;
2050                 udelay(1000);
2051         }
2052
2053         return 0;
2054 }
2055 #endif
2056
2057 static int ci_notify_smc_display_change(struct amdgpu_device *adev,
2058                                         bool has_display)
2059 {
2060         PPSMC_Msg msg = has_display ? PPSMC_MSG_HasDisplay : PPSMC_MSG_NoDisplay;
2061
2062         return (amdgpu_ci_send_msg_to_smc(adev, msg) == PPSMC_Result_OK) ?  0 : -EINVAL;
2063 }
2064
2065 static int ci_enable_ds_master_switch(struct amdgpu_device *adev,
2066                                       bool enable)
2067 {
2068         struct ci_power_info *pi = ci_get_pi(adev);
2069
2070         if (enable) {
2071                 if (pi->caps_sclk_ds) {
2072                         if (amdgpu_ci_send_msg_to_smc(adev, PPSMC_MSG_MASTER_DeepSleep_ON) != PPSMC_Result_OK)
2073                                 return -EINVAL;
2074                 } else {
2075                         if (amdgpu_ci_send_msg_to_smc(adev, PPSMC_MSG_MASTER_DeepSleep_OFF) != PPSMC_Result_OK)
2076                                 return -EINVAL;
2077                 }
2078         } else {
2079                 if (pi->caps_sclk_ds) {
2080                         if (amdgpu_ci_send_msg_to_smc(adev, PPSMC_MSG_MASTER_DeepSleep_OFF) != PPSMC_Result_OK)
2081                                 return -EINVAL;
2082                 }
2083         }
2084
2085         return 0;
2086 }
2087
2088 static void ci_program_display_gap(struct amdgpu_device *adev)
2089 {
2090         u32 tmp = RREG32_SMC(ixCG_DISPLAY_GAP_CNTL);
2091         u32 pre_vbi_time_in_us;
2092         u32 frame_time_in_us;
2093         u32 ref_clock = adev->clock.spll.reference_freq;
2094         u32 refresh_rate = amdgpu_dpm_get_vrefresh(adev);
2095         u32 vblank_time = amdgpu_dpm_get_vblank_time(adev);
2096
2097         tmp &= ~CG_DISPLAY_GAP_CNTL__DISP_GAP_MASK;
2098         if (adev->pm.dpm.new_active_crtc_count > 0)
2099                 tmp |= (AMDGPU_PM_DISPLAY_GAP_VBLANK_OR_WM << CG_DISPLAY_GAP_CNTL__DISP_GAP__SHIFT);
2100         else
2101                 tmp |= (AMDGPU_PM_DISPLAY_GAP_IGNORE << CG_DISPLAY_GAP_CNTL__DISP_GAP__SHIFT);
2102         WREG32_SMC(ixCG_DISPLAY_GAP_CNTL, tmp);
2103
2104         if (refresh_rate == 0)
2105                 refresh_rate = 60;
2106         if (vblank_time == 0xffffffff)
2107                 vblank_time = 500;
2108         frame_time_in_us = 1000000 / refresh_rate;
2109         pre_vbi_time_in_us =
2110                 frame_time_in_us - 200 - vblank_time;
2111         tmp = pre_vbi_time_in_us * (ref_clock / 100);
2112
2113         WREG32_SMC(ixCG_DISPLAY_GAP_CNTL2, tmp);
2114         ci_write_smc_soft_register(adev, offsetof(SMU7_SoftRegisters, PreVBlankGap), 0x64);
2115         ci_write_smc_soft_register(adev, offsetof(SMU7_SoftRegisters, VBlankTimeout), (frame_time_in_us - pre_vbi_time_in_us));
2116
2117
2118         ci_notify_smc_display_change(adev, (adev->pm.dpm.new_active_crtc_count == 1));
2119
2120 }
2121
2122 static void ci_enable_spread_spectrum(struct amdgpu_device *adev, bool enable)
2123 {
2124         struct ci_power_info *pi = ci_get_pi(adev);
2125         u32 tmp;
2126
2127         if (enable) {
2128                 if (pi->caps_sclk_ss_support) {
2129                         tmp = RREG32_SMC(ixGENERAL_PWRMGT);
2130                         tmp |= GENERAL_PWRMGT__DYN_SPREAD_SPECTRUM_EN_MASK;
2131                         WREG32_SMC(ixGENERAL_PWRMGT, tmp);
2132                 }
2133         } else {
2134                 tmp = RREG32_SMC(ixCG_SPLL_SPREAD_SPECTRUM);
2135                 tmp &= ~CG_SPLL_SPREAD_SPECTRUM__SSEN_MASK;
2136                 WREG32_SMC(ixCG_SPLL_SPREAD_SPECTRUM, tmp);
2137
2138                 tmp = RREG32_SMC(ixGENERAL_PWRMGT);
2139                 tmp &= ~GENERAL_PWRMGT__DYN_SPREAD_SPECTRUM_EN_MASK;
2140                 WREG32_SMC(ixGENERAL_PWRMGT, tmp);
2141         }
2142 }
2143
2144 static void ci_program_sstp(struct amdgpu_device *adev)
2145 {
2146         WREG32_SMC(ixCG_STATIC_SCREEN_PARAMETER,
2147         ((CISLANDS_SSTU_DFLT << CG_STATIC_SCREEN_PARAMETER__STATIC_SCREEN_THRESHOLD_UNIT__SHIFT) |
2148          (CISLANDS_SST_DFLT << CG_STATIC_SCREEN_PARAMETER__STATIC_SCREEN_THRESHOLD__SHIFT)));
2149 }
2150
2151 static void ci_enable_display_gap(struct amdgpu_device *adev)
2152 {
2153         u32 tmp = RREG32_SMC(ixCG_DISPLAY_GAP_CNTL);
2154
2155         tmp &= ~(CG_DISPLAY_GAP_CNTL__DISP_GAP_MASK |
2156                         CG_DISPLAY_GAP_CNTL__DISP_GAP_MCHG_MASK);
2157         tmp |= ((AMDGPU_PM_DISPLAY_GAP_IGNORE << CG_DISPLAY_GAP_CNTL__DISP_GAP__SHIFT) |
2158                 (AMDGPU_PM_DISPLAY_GAP_VBLANK << CG_DISPLAY_GAP_CNTL__DISP_GAP_MCHG__SHIFT));
2159
2160         WREG32_SMC(ixCG_DISPLAY_GAP_CNTL, tmp);
2161 }
2162
2163 static void ci_program_vc(struct amdgpu_device *adev)
2164 {
2165         u32 tmp;
2166
2167         tmp = RREG32_SMC(ixSCLK_PWRMGT_CNTL);
2168         tmp &= ~(SCLK_PWRMGT_CNTL__RESET_SCLK_CNT_MASK | SCLK_PWRMGT_CNTL__RESET_BUSY_CNT_MASK);
2169         WREG32_SMC(ixSCLK_PWRMGT_CNTL, tmp);
2170
2171         WREG32_SMC(ixCG_FREQ_TRAN_VOTING_0, CISLANDS_VRC_DFLT0);
2172         WREG32_SMC(ixCG_FREQ_TRAN_VOTING_1, CISLANDS_VRC_DFLT1);
2173         WREG32_SMC(ixCG_FREQ_TRAN_VOTING_2, CISLANDS_VRC_DFLT2);
2174         WREG32_SMC(ixCG_FREQ_TRAN_VOTING_3, CISLANDS_VRC_DFLT3);
2175         WREG32_SMC(ixCG_FREQ_TRAN_VOTING_4, CISLANDS_VRC_DFLT4);
2176         WREG32_SMC(ixCG_FREQ_TRAN_VOTING_5, CISLANDS_VRC_DFLT5);
2177         WREG32_SMC(ixCG_FREQ_TRAN_VOTING_6, CISLANDS_VRC_DFLT6);
2178         WREG32_SMC(ixCG_FREQ_TRAN_VOTING_7, CISLANDS_VRC_DFLT7);
2179 }
2180
2181 static void ci_clear_vc(struct amdgpu_device *adev)
2182 {
2183         u32 tmp;
2184
2185         tmp = RREG32_SMC(ixSCLK_PWRMGT_CNTL);
2186         tmp |= (SCLK_PWRMGT_CNTL__RESET_SCLK_CNT_MASK | SCLK_PWRMGT_CNTL__RESET_BUSY_CNT_MASK);
2187         WREG32_SMC(ixSCLK_PWRMGT_CNTL, tmp);
2188
2189         WREG32_SMC(ixCG_FREQ_TRAN_VOTING_0, 0);
2190         WREG32_SMC(ixCG_FREQ_TRAN_VOTING_1, 0);
2191         WREG32_SMC(ixCG_FREQ_TRAN_VOTING_2, 0);
2192         WREG32_SMC(ixCG_FREQ_TRAN_VOTING_3, 0);
2193         WREG32_SMC(ixCG_FREQ_TRAN_VOTING_4, 0);
2194         WREG32_SMC(ixCG_FREQ_TRAN_VOTING_5, 0);
2195         WREG32_SMC(ixCG_FREQ_TRAN_VOTING_6, 0);
2196         WREG32_SMC(ixCG_FREQ_TRAN_VOTING_7, 0);
2197 }
2198
2199 static int ci_upload_firmware(struct amdgpu_device *adev)
2200 {
2201         struct ci_power_info *pi = ci_get_pi(adev);
2202         int i, ret;
2203
2204         for (i = 0; i < adev->usec_timeout; i++) {
2205                 if (RREG32_SMC(ixRCU_UC_EVENTS) & RCU_UC_EVENTS__boot_seq_done_MASK)
2206                         break;
2207         }
2208         WREG32_SMC(ixSMC_SYSCON_MISC_CNTL, 1);
2209
2210         amdgpu_ci_stop_smc_clock(adev);
2211         amdgpu_ci_reset_smc(adev);
2212
2213         ret = amdgpu_ci_load_smc_ucode(adev, pi->sram_end);
2214
2215         return ret;
2216
2217 }
2218
2219 static int ci_get_svi2_voltage_table(struct amdgpu_device *adev,
2220                                      struct amdgpu_clock_voltage_dependency_table *voltage_dependency_table,
2221                                      struct atom_voltage_table *voltage_table)
2222 {
2223         u32 i;
2224
2225         if (voltage_dependency_table == NULL)
2226                 return -EINVAL;
2227
2228         voltage_table->mask_low = 0;
2229         voltage_table->phase_delay = 0;
2230
2231         voltage_table->count = voltage_dependency_table->count;
2232         for (i = 0; i < voltage_table->count; i++) {
2233                 voltage_table->entries[i].value = voltage_dependency_table->entries[i].v;
2234                 voltage_table->entries[i].smio_low = 0;
2235         }
2236
2237         return 0;
2238 }
2239
2240 static int ci_construct_voltage_tables(struct amdgpu_device *adev)
2241 {
2242         struct ci_power_info *pi = ci_get_pi(adev);
2243         int ret;
2244
2245         if (pi->voltage_control == CISLANDS_VOLTAGE_CONTROL_BY_GPIO) {
2246                 ret = amdgpu_atombios_get_voltage_table(adev, VOLTAGE_TYPE_VDDC,
2247                                                         VOLTAGE_OBJ_GPIO_LUT,
2248                                                         &pi->vddc_voltage_table);
2249                 if (ret)
2250                         return ret;
2251         } else if (pi->voltage_control == CISLANDS_VOLTAGE_CONTROL_BY_SVID2) {
2252                 ret = ci_get_svi2_voltage_table(adev,
2253                                                 &adev->pm.dpm.dyn_state.vddc_dependency_on_mclk,
2254                                                 &pi->vddc_voltage_table);
2255                 if (ret)
2256                         return ret;
2257         }
2258
2259         if (pi->vddc_voltage_table.count > SMU7_MAX_LEVELS_VDDC)
2260                 ci_trim_voltage_table_to_fit_state_table(adev, SMU7_MAX_LEVELS_VDDC,
2261                                                          &pi->vddc_voltage_table);
2262
2263         if (pi->vddci_control == CISLANDS_VOLTAGE_CONTROL_BY_GPIO) {
2264                 ret = amdgpu_atombios_get_voltage_table(adev, VOLTAGE_TYPE_VDDCI,
2265                                                         VOLTAGE_OBJ_GPIO_LUT,
2266                                                         &pi->vddci_voltage_table);
2267                 if (ret)
2268                         return ret;
2269         } else if (pi->vddci_control == CISLANDS_VOLTAGE_CONTROL_BY_SVID2) {
2270                 ret = ci_get_svi2_voltage_table(adev,
2271                                                 &adev->pm.dpm.dyn_state.vddci_dependency_on_mclk,
2272                                                 &pi->vddci_voltage_table);
2273                 if (ret)
2274                         return ret;
2275         }
2276
2277         if (pi->vddci_voltage_table.count > SMU7_MAX_LEVELS_VDDCI)
2278                 ci_trim_voltage_table_to_fit_state_table(adev, SMU7_MAX_LEVELS_VDDCI,
2279                                                          &pi->vddci_voltage_table);
2280
2281         if (pi->mvdd_control == CISLANDS_VOLTAGE_CONTROL_BY_GPIO) {
2282                 ret = amdgpu_atombios_get_voltage_table(adev, VOLTAGE_TYPE_MVDDC,
2283                                                         VOLTAGE_OBJ_GPIO_LUT,
2284                                                         &pi->mvdd_voltage_table);
2285                 if (ret)
2286                         return ret;
2287         } else if (pi->mvdd_control == CISLANDS_VOLTAGE_CONTROL_BY_SVID2) {
2288                 ret = ci_get_svi2_voltage_table(adev,
2289                                                 &adev->pm.dpm.dyn_state.mvdd_dependency_on_mclk,
2290                                                 &pi->mvdd_voltage_table);
2291                 if (ret)
2292                         return ret;
2293         }
2294
2295         if (pi->mvdd_voltage_table.count > SMU7_MAX_LEVELS_MVDD)
2296                 ci_trim_voltage_table_to_fit_state_table(adev, SMU7_MAX_LEVELS_MVDD,
2297                                                          &pi->mvdd_voltage_table);
2298
2299         return 0;
2300 }
2301
2302 static void ci_populate_smc_voltage_table(struct amdgpu_device *adev,
2303                                           struct atom_voltage_table_entry *voltage_table,
2304                                           SMU7_Discrete_VoltageLevel *smc_voltage_table)
2305 {
2306         int ret;
2307
2308         ret = ci_get_std_voltage_value_sidd(adev, voltage_table,
2309                                             &smc_voltage_table->StdVoltageHiSidd,
2310                                             &smc_voltage_table->StdVoltageLoSidd);
2311
2312         if (ret) {
2313                 smc_voltage_table->StdVoltageHiSidd = voltage_table->value * VOLTAGE_SCALE;
2314                 smc_voltage_table->StdVoltageLoSidd = voltage_table->value * VOLTAGE_SCALE;
2315         }
2316
2317         smc_voltage_table->Voltage = cpu_to_be16(voltage_table->value * VOLTAGE_SCALE);
2318         smc_voltage_table->StdVoltageHiSidd =
2319                 cpu_to_be16(smc_voltage_table->StdVoltageHiSidd);
2320         smc_voltage_table->StdVoltageLoSidd =
2321                 cpu_to_be16(smc_voltage_table->StdVoltageLoSidd);
2322 }
2323
2324 static int ci_populate_smc_vddc_table(struct amdgpu_device *adev,
2325                                       SMU7_Discrete_DpmTable *table)
2326 {
2327         struct ci_power_info *pi = ci_get_pi(adev);
2328         unsigned int count;
2329
2330         table->VddcLevelCount = pi->vddc_voltage_table.count;
2331         for (count = 0; count < table->VddcLevelCount; count++) {
2332                 ci_populate_smc_voltage_table(adev,
2333                                               &pi->vddc_voltage_table.entries[count],
2334                                               &table->VddcLevel[count]);
2335
2336                 if (pi->voltage_control == CISLANDS_VOLTAGE_CONTROL_BY_GPIO)
2337                         table->VddcLevel[count].Smio |=
2338                                 pi->vddc_voltage_table.entries[count].smio_low;
2339                 else
2340                         table->VddcLevel[count].Smio = 0;
2341         }
2342         table->VddcLevelCount = cpu_to_be32(table->VddcLevelCount);
2343
2344         return 0;
2345 }
2346
2347 static int ci_populate_smc_vddci_table(struct amdgpu_device *adev,
2348                                        SMU7_Discrete_DpmTable *table)
2349 {
2350         unsigned int count;
2351         struct ci_power_info *pi = ci_get_pi(adev);
2352
2353         table->VddciLevelCount = pi->vddci_voltage_table.count;
2354         for (count = 0; count < table->VddciLevelCount; count++) {
2355                 ci_populate_smc_voltage_table(adev,
2356                                               &pi->vddci_voltage_table.entries[count],
2357                                               &table->VddciLevel[count]);
2358
2359                 if (pi->vddci_control == CISLANDS_VOLTAGE_CONTROL_BY_GPIO)
2360                         table->VddciLevel[count].Smio |=
2361                                 pi->vddci_voltage_table.entries[count].smio_low;
2362                 else
2363                         table->VddciLevel[count].Smio = 0;
2364         }
2365         table->VddciLevelCount = cpu_to_be32(table->VddciLevelCount);
2366
2367         return 0;
2368 }
2369
2370 static int ci_populate_smc_mvdd_table(struct amdgpu_device *adev,
2371                                       SMU7_Discrete_DpmTable *table)
2372 {
2373         struct ci_power_info *pi = ci_get_pi(adev);
2374         unsigned int count;
2375
2376         table->MvddLevelCount = pi->mvdd_voltage_table.count;
2377         for (count = 0; count < table->MvddLevelCount; count++) {
2378                 ci_populate_smc_voltage_table(adev,
2379                                               &pi->mvdd_voltage_table.entries[count],
2380                                               &table->MvddLevel[count]);
2381
2382                 if (pi->mvdd_control == CISLANDS_VOLTAGE_CONTROL_BY_GPIO)
2383                         table->MvddLevel[count].Smio |=
2384                                 pi->mvdd_voltage_table.entries[count].smio_low;
2385                 else
2386                         table->MvddLevel[count].Smio = 0;
2387         }
2388         table->MvddLevelCount = cpu_to_be32(table->MvddLevelCount);
2389
2390         return 0;
2391 }
2392
2393 static int ci_populate_smc_voltage_tables(struct amdgpu_device *adev,
2394                                           SMU7_Discrete_DpmTable *table)
2395 {
2396         int ret;
2397
2398         ret = ci_populate_smc_vddc_table(adev, table);
2399         if (ret)
2400                 return ret;
2401
2402         ret = ci_populate_smc_vddci_table(adev, table);
2403         if (ret)
2404                 return ret;
2405
2406         ret = ci_populate_smc_mvdd_table(adev, table);
2407         if (ret)
2408                 return ret;
2409
2410         return 0;
2411 }
2412
2413 static int ci_populate_mvdd_value(struct amdgpu_device *adev, u32 mclk,
2414                                   SMU7_Discrete_VoltageLevel *voltage)
2415 {
2416         struct ci_power_info *pi = ci_get_pi(adev);
2417         u32 i = 0;
2418
2419         if (pi->mvdd_control != CISLANDS_VOLTAGE_CONTROL_NONE) {
2420                 for (i = 0; i < adev->pm.dpm.dyn_state.mvdd_dependency_on_mclk.count; i++) {
2421                         if (mclk <= adev->pm.dpm.dyn_state.mvdd_dependency_on_mclk.entries[i].clk) {
2422                                 voltage->Voltage = pi->mvdd_voltage_table.entries[i].value;
2423                                 break;
2424                         }
2425                 }
2426
2427                 if (i >= adev->pm.dpm.dyn_state.mvdd_dependency_on_mclk.count)
2428                         return -EINVAL;
2429         }
2430
2431         return -EINVAL;
2432 }
2433
2434 static int ci_get_std_voltage_value_sidd(struct amdgpu_device *adev,
2435                                          struct atom_voltage_table_entry *voltage_table,
2436                                          u16 *std_voltage_hi_sidd, u16 *std_voltage_lo_sidd)
2437 {
2438         u16 v_index, idx;
2439         bool voltage_found = false;
2440         *std_voltage_hi_sidd = voltage_table->value * VOLTAGE_SCALE;
2441         *std_voltage_lo_sidd = voltage_table->value * VOLTAGE_SCALE;
2442
2443         if (adev->pm.dpm.dyn_state.vddc_dependency_on_sclk.entries == NULL)
2444                 return -EINVAL;
2445
2446         if (adev->pm.dpm.dyn_state.cac_leakage_table.entries) {
2447                 for (v_index = 0; (u32)v_index < adev->pm.dpm.dyn_state.vddc_dependency_on_sclk.count; v_index++) {
2448                         if (voltage_table->value ==
2449                             adev->pm.dpm.dyn_state.vddc_dependency_on_sclk.entries[v_index].v) {
2450                                 voltage_found = true;
2451                                 if ((u32)v_index < adev->pm.dpm.dyn_state.cac_leakage_table.count)
2452                                         idx = v_index;
2453                                 else
2454                                         idx = adev->pm.dpm.dyn_state.cac_leakage_table.count - 1;
2455                                 *std_voltage_lo_sidd =
2456                                         adev->pm.dpm.dyn_state.cac_leakage_table.entries[idx].vddc * VOLTAGE_SCALE;
2457                                 *std_voltage_hi_sidd =
2458                                         adev->pm.dpm.dyn_state.cac_leakage_table.entries[idx].leakage * VOLTAGE_SCALE;
2459                                 break;
2460                         }
2461                 }
2462
2463                 if (!voltage_found) {
2464                         for (v_index = 0; (u32)v_index < adev->pm.dpm.dyn_state.vddc_dependency_on_sclk.count; v_index++) {
2465                                 if (voltage_table->value <=
2466                                     adev->pm.dpm.dyn_state.vddc_dependency_on_sclk.entries[v_index].v) {
2467                                         voltage_found = true;
2468                                         if ((u32)v_index < adev->pm.dpm.dyn_state.cac_leakage_table.count)
2469                                                 idx = v_index;
2470                                         else
2471                                                 idx = adev->pm.dpm.dyn_state.cac_leakage_table.count - 1;
2472                                         *std_voltage_lo_sidd =
2473                                                 adev->pm.dpm.dyn_state.cac_leakage_table.entries[idx].vddc * VOLTAGE_SCALE;
2474                                         *std_voltage_hi_sidd =
2475                                                 adev->pm.dpm.dyn_state.cac_leakage_table.entries[idx].leakage * VOLTAGE_SCALE;
2476                                         break;
2477                                 }
2478                         }
2479                 }
2480         }
2481
2482         return 0;
2483 }
2484
2485 static void ci_populate_phase_value_based_on_sclk(struct amdgpu_device *adev,
2486                                                   const struct amdgpu_phase_shedding_limits_table *limits,
2487                                                   u32 sclk,
2488                                                   u32 *phase_shedding)
2489 {
2490         unsigned int i;
2491
2492         *phase_shedding = 1;
2493
2494         for (i = 0; i < limits->count; i++) {
2495                 if (sclk < limits->entries[i].sclk) {
2496                         *phase_shedding = i;
2497                         break;
2498                 }
2499         }
2500 }
2501
2502 static void ci_populate_phase_value_based_on_mclk(struct amdgpu_device *adev,
2503                                                   const struct amdgpu_phase_shedding_limits_table *limits,
2504                                                   u32 mclk,
2505                                                   u32 *phase_shedding)
2506 {
2507         unsigned int i;
2508
2509         *phase_shedding = 1;
2510
2511         for (i = 0; i < limits->count; i++) {
2512                 if (mclk < limits->entries[i].mclk) {
2513                         *phase_shedding = i;
2514                         break;
2515                 }
2516         }
2517 }
2518
2519 static int ci_init_arb_table_index(struct amdgpu_device *adev)
2520 {
2521         struct ci_power_info *pi = ci_get_pi(adev);
2522         u32 tmp;
2523         int ret;
2524
2525         ret = amdgpu_ci_read_smc_sram_dword(adev, pi->arb_table_start,
2526                                      &tmp, pi->sram_end);
2527         if (ret)
2528                 return ret;
2529
2530         tmp &= 0x00FFFFFF;
2531         tmp |= MC_CG_ARB_FREQ_F1 << 24;
2532
2533         return amdgpu_ci_write_smc_sram_dword(adev, pi->arb_table_start,
2534                                        tmp, pi->sram_end);
2535 }
2536
2537 static int ci_get_dependency_volt_by_clk(struct amdgpu_device *adev,
2538                                          struct amdgpu_clock_voltage_dependency_table *allowed_clock_voltage_table,
2539                                          u32 clock, u32 *voltage)
2540 {
2541         u32 i = 0;
2542
2543         if (allowed_clock_voltage_table->count == 0)
2544                 return -EINVAL;
2545
2546         for (i = 0; i < allowed_clock_voltage_table->count; i++) {
2547                 if (allowed_clock_voltage_table->entries[i].clk >= clock) {
2548                         *voltage = allowed_clock_voltage_table->entries[i].v;
2549                         return 0;
2550                 }
2551         }
2552
2553         *voltage = allowed_clock_voltage_table->entries[i-1].v;
2554
2555         return 0;
2556 }
2557
2558 static u8 ci_get_sleep_divider_id_from_clock(struct amdgpu_device *adev,
2559                                              u32 sclk, u32 min_sclk_in_sr)
2560 {
2561         u32 i;
2562         u32 tmp;
2563         u32 min = (min_sclk_in_sr > CISLAND_MINIMUM_ENGINE_CLOCK) ?
2564                 min_sclk_in_sr : CISLAND_MINIMUM_ENGINE_CLOCK;
2565
2566         if (sclk < min)
2567                 return 0;
2568
2569         for (i = CISLAND_MAX_DEEPSLEEP_DIVIDER_ID;  ; i--) {
2570                 tmp = sclk / (1 << i);
2571                 if (tmp >= min || i == 0)
2572                         break;
2573         }
2574
2575         return (u8)i;
2576 }
2577
2578 static int ci_initial_switch_from_arb_f0_to_f1(struct amdgpu_device *adev)
2579 {
2580         return ci_copy_and_switch_arb_sets(adev, MC_CG_ARB_FREQ_F0, MC_CG_ARB_FREQ_F1);
2581 }
2582
2583 static int ci_reset_to_default(struct amdgpu_device *adev)
2584 {
2585         return (amdgpu_ci_send_msg_to_smc(adev, PPSMC_MSG_ResetToDefaults) == PPSMC_Result_OK) ?
2586                 0 : -EINVAL;
2587 }
2588
2589 static int ci_force_switch_to_arb_f0(struct amdgpu_device *adev)
2590 {
2591         u32 tmp;
2592
2593         tmp = (RREG32_SMC(ixSMC_SCRATCH9) & 0x0000ff00) >> 8;
2594
2595         if (tmp == MC_CG_ARB_FREQ_F0)
2596                 return 0;
2597
2598         return ci_copy_and_switch_arb_sets(adev, tmp, MC_CG_ARB_FREQ_F0);
2599 }
2600
2601 static void ci_register_patching_mc_arb(struct amdgpu_device *adev,
2602                                         const u32 engine_clock,
2603                                         const u32 memory_clock,
2604                                         u32 *dram_timimg2)
2605 {
2606         bool patch;
2607         u32 tmp, tmp2;
2608
2609         tmp = RREG32(mmMC_SEQ_MISC0);
2610         patch = ((tmp & 0x0000f00) == 0x300) ? true : false;
2611
2612         if (patch &&
2613             ((adev->pdev->device == 0x67B0) ||
2614              (adev->pdev->device == 0x67B1))) {
2615                 if ((memory_clock > 100000) && (memory_clock <= 125000)) {
2616                         tmp2 = (((0x31 * engine_clock) / 125000) - 1) & 0xff;
2617                         *dram_timimg2 &= ~0x00ff0000;
2618                         *dram_timimg2 |= tmp2 << 16;
2619                 } else if ((memory_clock > 125000) && (memory_clock <= 137500)) {
2620                         tmp2 = (((0x36 * engine_clock) / 137500) - 1) & 0xff;
2621                         *dram_timimg2 &= ~0x00ff0000;
2622                         *dram_timimg2 |= tmp2 << 16;
2623                 }
2624         }
2625 }
2626
2627 static int ci_populate_memory_timing_parameters(struct amdgpu_device *adev,
2628                                                 u32 sclk,
2629                                                 u32 mclk,
2630                                                 SMU7_Discrete_MCArbDramTimingTableEntry *arb_regs)
2631 {
2632         u32 dram_timing;
2633         u32 dram_timing2;
2634         u32 burst_time;
2635
2636         amdgpu_atombios_set_engine_dram_timings(adev, sclk, mclk);
2637
2638         dram_timing  = RREG32(mmMC_ARB_DRAM_TIMING);
2639         dram_timing2 = RREG32(mmMC_ARB_DRAM_TIMING2);
2640         burst_time = RREG32(mmMC_ARB_BURST_TIME) & MC_ARB_BURST_TIME__STATE0_MASK;
2641
2642         ci_register_patching_mc_arb(adev, sclk, mclk, &dram_timing2);
2643
2644         arb_regs->McArbDramTiming  = cpu_to_be32(dram_timing);
2645         arb_regs->McArbDramTiming2 = cpu_to_be32(dram_timing2);
2646         arb_regs->McArbBurstTime = (u8)burst_time;
2647
2648         return 0;
2649 }
2650
2651 static int ci_do_program_memory_timing_parameters(struct amdgpu_device *adev)
2652 {
2653         struct ci_power_info *pi = ci_get_pi(adev);
2654         SMU7_Discrete_MCArbDramTimingTable arb_regs;
2655         u32 i, j;
2656         int ret =  0;
2657
2658         memset(&arb_regs, 0, sizeof(SMU7_Discrete_MCArbDramTimingTable));
2659
2660         for (i = 0; i < pi->dpm_table.sclk_table.count; i++) {
2661                 for (j = 0; j < pi->dpm_table.mclk_table.count; j++) {
2662                         ret = ci_populate_memory_timing_parameters(adev,
2663                                                                    pi->dpm_table.sclk_table.dpm_levels[i].value,
2664                                                                    pi->dpm_table.mclk_table.dpm_levels[j].value,
2665                                                                    &arb_regs.entries[i][j]);
2666                         if (ret)
2667                                 break;
2668                 }
2669         }
2670
2671         if (ret == 0)
2672                 ret = amdgpu_ci_copy_bytes_to_smc(adev,
2673                                            pi->arb_table_start,
2674                                            (u8 *)&arb_regs,
2675                                            sizeof(SMU7_Discrete_MCArbDramTimingTable),
2676                                            pi->sram_end);
2677
2678         return ret;
2679 }
2680
2681 static int ci_program_memory_timing_parameters(struct amdgpu_device *adev)
2682 {
2683         struct ci_power_info *pi = ci_get_pi(adev);
2684
2685         if (pi->need_update_smu7_dpm_table == 0)
2686                 return 0;
2687
2688         return ci_do_program_memory_timing_parameters(adev);
2689 }
2690
2691 static void ci_populate_smc_initial_state(struct amdgpu_device *adev,
2692                                           struct amdgpu_ps *amdgpu_boot_state)
2693 {
2694         struct ci_ps *boot_state = ci_get_ps(amdgpu_boot_state);
2695         struct ci_power_info *pi = ci_get_pi(adev);
2696         u32 level = 0;
2697
2698         for (level = 0; level < adev->pm.dpm.dyn_state.vddc_dependency_on_sclk.count; level++) {
2699                 if (adev->pm.dpm.dyn_state.vddc_dependency_on_sclk.entries[level].clk >=
2700                     boot_state->performance_levels[0].sclk) {
2701                         pi->smc_state_table.GraphicsBootLevel = level;
2702                         break;
2703                 }
2704         }
2705
2706         for (level = 0; level < adev->pm.dpm.dyn_state.vddc_dependency_on_mclk.count; level++) {
2707                 if (adev->pm.dpm.dyn_state.vddc_dependency_on_mclk.entries[level].clk >=
2708                     boot_state->performance_levels[0].mclk) {
2709                         pi->smc_state_table.MemoryBootLevel = level;
2710                         break;
2711                 }
2712         }
2713 }
2714
2715 static u32 ci_get_dpm_level_enable_mask_value(struct ci_single_dpm_table *dpm_table)
2716 {
2717         u32 i;
2718         u32 mask_value = 0;
2719
2720         for (i = dpm_table->count; i > 0; i--) {
2721                 mask_value = mask_value << 1;
2722                 if (dpm_table->dpm_levels[i-1].enabled)
2723                         mask_value |= 0x1;
2724                 else
2725                         mask_value &= 0xFFFFFFFE;
2726         }
2727
2728         return mask_value;
2729 }
2730
2731 static void ci_populate_smc_link_level(struct amdgpu_device *adev,
2732                                        SMU7_Discrete_DpmTable *table)
2733 {
2734         struct ci_power_info *pi = ci_get_pi(adev);
2735         struct ci_dpm_table *dpm_table = &pi->dpm_table;
2736         u32 i;
2737
2738         for (i = 0; i < dpm_table->pcie_speed_table.count; i++) {
2739                 table->LinkLevel[i].PcieGenSpeed =
2740                         (u8)dpm_table->pcie_speed_table.dpm_levels[i].value;
2741                 table->LinkLevel[i].PcieLaneCount =
2742                         amdgpu_encode_pci_lane_width(dpm_table->pcie_speed_table.dpm_levels[i].param1);
2743                 table->LinkLevel[i].EnabledForActivity = 1;
2744                 table->LinkLevel[i].DownT = cpu_to_be32(5);
2745                 table->LinkLevel[i].UpT = cpu_to_be32(30);
2746         }
2747
2748         pi->smc_state_table.LinkLevelCount = (u8)dpm_table->pcie_speed_table.count;
2749         pi->dpm_level_enable_mask.pcie_dpm_enable_mask =
2750                 ci_get_dpm_level_enable_mask_value(&dpm_table->pcie_speed_table);
2751 }
2752
2753 static int ci_populate_smc_uvd_level(struct amdgpu_device *adev,
2754                                      SMU7_Discrete_DpmTable *table)
2755 {
2756         u32 count;
2757         struct atom_clock_dividers dividers;
2758         int ret = -EINVAL;
2759
2760         table->UvdLevelCount =
2761                 adev->pm.dpm.dyn_state.uvd_clock_voltage_dependency_table.count;
2762
2763         for (count = 0; count < table->UvdLevelCount; count++) {
2764                 table->UvdLevel[count].VclkFrequency =
2765                         adev->pm.dpm.dyn_state.uvd_clock_voltage_dependency_table.entries[count].vclk;
2766                 table->UvdLevel[count].DclkFrequency =
2767                         adev->pm.dpm.dyn_state.uvd_clock_voltage_dependency_table.entries[count].dclk;
2768                 table->UvdLevel[count].MinVddc =
2769                         adev->pm.dpm.dyn_state.uvd_clock_voltage_dependency_table.entries[count].v * VOLTAGE_SCALE;
2770                 table->UvdLevel[count].MinVddcPhases = 1;
2771
2772                 ret = amdgpu_atombios_get_clock_dividers(adev,
2773                                                          COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
2774                                                          table->UvdLevel[count].VclkFrequency, false, &dividers);
2775                 if (ret)
2776                         return ret;
2777
2778                 table->UvdLevel[count].VclkDivider = (u8)dividers.post_divider;
2779
2780                 ret = amdgpu_atombios_get_clock_dividers(adev,
2781                                                          COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
2782                                                          table->UvdLevel[count].DclkFrequency, false, &dividers);
2783                 if (ret)
2784                         return ret;
2785
2786                 table->UvdLevel[count].DclkDivider = (u8)dividers.post_divider;
2787
2788                 table->UvdLevel[count].VclkFrequency = cpu_to_be32(table->UvdLevel[count].VclkFrequency);
2789                 table->UvdLevel[count].DclkFrequency = cpu_to_be32(table->UvdLevel[count].DclkFrequency);
2790                 table->UvdLevel[count].MinVddc = cpu_to_be16(table->UvdLevel[count].MinVddc);
2791         }
2792
2793         return ret;
2794 }
2795
2796 static int ci_populate_smc_vce_level(struct amdgpu_device *adev,
2797                                      SMU7_Discrete_DpmTable *table)
2798 {
2799         u32 count;
2800         struct atom_clock_dividers dividers;
2801         int ret = -EINVAL;
2802
2803         table->VceLevelCount =
2804                 adev->pm.dpm.dyn_state.vce_clock_voltage_dependency_table.count;
2805
2806         for (count = 0; count < table->VceLevelCount; count++) {
2807                 table->VceLevel[count].Frequency =
2808                         adev->pm.dpm.dyn_state.vce_clock_voltage_dependency_table.entries[count].evclk;
2809                 table->VceLevel[count].MinVoltage =
2810                         (u16)adev->pm.dpm.dyn_state.vce_clock_voltage_dependency_table.entries[count].v * VOLTAGE_SCALE;
2811                 table->VceLevel[count].MinPhases = 1;
2812
2813                 ret = amdgpu_atombios_get_clock_dividers(adev,
2814                                                          COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
2815                                                          table->VceLevel[count].Frequency, false, &dividers);
2816                 if (ret)
2817                         return ret;
2818
2819                 table->VceLevel[count].Divider = (u8)dividers.post_divider;
2820
2821                 table->VceLevel[count].Frequency = cpu_to_be32(table->VceLevel[count].Frequency);
2822                 table->VceLevel[count].MinVoltage = cpu_to_be16(table->VceLevel[count].MinVoltage);
2823         }
2824
2825         return ret;
2826
2827 }
2828
2829 static int ci_populate_smc_acp_level(struct amdgpu_device *adev,
2830                                      SMU7_Discrete_DpmTable *table)
2831 {
2832         u32 count;
2833         struct atom_clock_dividers dividers;
2834         int ret = -EINVAL;
2835
2836         table->AcpLevelCount = (u8)
2837                 (adev->pm.dpm.dyn_state.acp_clock_voltage_dependency_table.count);
2838
2839         for (count = 0; count < table->AcpLevelCount; count++) {
2840                 table->AcpLevel[count].Frequency =
2841                         adev->pm.dpm.dyn_state.acp_clock_voltage_dependency_table.entries[count].clk;
2842                 table->AcpLevel[count].MinVoltage =
2843                         adev->pm.dpm.dyn_state.acp_clock_voltage_dependency_table.entries[count].v;
2844                 table->AcpLevel[count].MinPhases = 1;
2845
2846                 ret = amdgpu_atombios_get_clock_dividers(adev,
2847                                                          COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
2848                                                          table->AcpLevel[count].Frequency, false, &dividers);
2849                 if (ret)
2850                         return ret;
2851
2852                 table->AcpLevel[count].Divider = (u8)dividers.post_divider;
2853
2854                 table->AcpLevel[count].Frequency = cpu_to_be32(table->AcpLevel[count].Frequency);
2855                 table->AcpLevel[count].MinVoltage = cpu_to_be16(table->AcpLevel[count].MinVoltage);
2856         }
2857
2858         return ret;
2859 }
2860
2861 static int ci_populate_smc_samu_level(struct amdgpu_device *adev,
2862                                       SMU7_Discrete_DpmTable *table)
2863 {
2864         u32 count;
2865         struct atom_clock_dividers dividers;
2866         int ret = -EINVAL;
2867
2868         table->SamuLevelCount =
2869                 adev->pm.dpm.dyn_state.samu_clock_voltage_dependency_table.count;
2870
2871         for (count = 0; count < table->SamuLevelCount; count++) {
2872                 table->SamuLevel[count].Frequency =
2873                         adev->pm.dpm.dyn_state.samu_clock_voltage_dependency_table.entries[count].clk;
2874                 table->SamuLevel[count].MinVoltage =
2875                         adev->pm.dpm.dyn_state.samu_clock_voltage_dependency_table.entries[count].v * VOLTAGE_SCALE;
2876                 table->SamuLevel[count].MinPhases = 1;
2877
2878                 ret = amdgpu_atombios_get_clock_dividers(adev,
2879                                                          COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
2880                                                          table->SamuLevel[count].Frequency, false, &dividers);
2881                 if (ret)
2882                         return ret;
2883
2884                 table->SamuLevel[count].Divider = (u8)dividers.post_divider;
2885
2886                 table->SamuLevel[count].Frequency = cpu_to_be32(table->SamuLevel[count].Frequency);
2887                 table->SamuLevel[count].MinVoltage = cpu_to_be16(table->SamuLevel[count].MinVoltage);
2888         }
2889
2890         return ret;
2891 }
2892
2893 static int ci_calculate_mclk_params(struct amdgpu_device *adev,
2894                                     u32 memory_clock,
2895                                     SMU7_Discrete_MemoryLevel *mclk,
2896                                     bool strobe_mode,
2897                                     bool dll_state_on)
2898 {
2899         struct ci_power_info *pi = ci_get_pi(adev);
2900         u32  dll_cntl = pi->clock_registers.dll_cntl;
2901         u32  mclk_pwrmgt_cntl = pi->clock_registers.mclk_pwrmgt_cntl;
2902         u32  mpll_ad_func_cntl = pi->clock_registers.mpll_ad_func_cntl;
2903         u32  mpll_dq_func_cntl = pi->clock_registers.mpll_dq_func_cntl;
2904         u32  mpll_func_cntl = pi->clock_registers.mpll_func_cntl;
2905         u32  mpll_func_cntl_1 = pi->clock_registers.mpll_func_cntl_1;
2906         u32  mpll_func_cntl_2 = pi->clock_registers.mpll_func_cntl_2;
2907         u32  mpll_ss1 = pi->clock_registers.mpll_ss1;
2908         u32  mpll_ss2 = pi->clock_registers.mpll_ss2;
2909         struct atom_mpll_param mpll_param;
2910         int ret;
2911
2912         ret = amdgpu_atombios_get_memory_pll_dividers(adev, memory_clock, strobe_mode, &mpll_param);
2913         if (ret)
2914                 return ret;
2915
2916         mpll_func_cntl &= ~MPLL_FUNC_CNTL__BWCTRL_MASK;
2917         mpll_func_cntl |= (mpll_param.bwcntl << MPLL_FUNC_CNTL__BWCTRL__SHIFT);
2918
2919         mpll_func_cntl_1 &= ~(MPLL_FUNC_CNTL_1__CLKF_MASK | MPLL_FUNC_CNTL_1__CLKFRAC_MASK |
2920                         MPLL_FUNC_CNTL_1__VCO_MODE_MASK);
2921         mpll_func_cntl_1 |= (mpll_param.clkf) << MPLL_FUNC_CNTL_1__CLKF__SHIFT |
2922                 (mpll_param.clkfrac << MPLL_FUNC_CNTL_1__CLKFRAC__SHIFT) |
2923                 (mpll_param.vco_mode << MPLL_FUNC_CNTL_1__VCO_MODE__SHIFT);
2924
2925         mpll_ad_func_cntl &= ~MPLL_AD_FUNC_CNTL__YCLK_POST_DIV_MASK;
2926         mpll_ad_func_cntl |= (mpll_param.post_div << MPLL_AD_FUNC_CNTL__YCLK_POST_DIV__SHIFT);
2927
2928         if (adev->mc.vram_type == AMDGPU_VRAM_TYPE_GDDR5) {
2929                 mpll_dq_func_cntl &= ~(MPLL_DQ_FUNC_CNTL__YCLK_SEL_MASK |
2930                                 MPLL_AD_FUNC_CNTL__YCLK_POST_DIV_MASK);
2931                 mpll_dq_func_cntl |= (mpll_param.yclk_sel << MPLL_DQ_FUNC_CNTL__YCLK_SEL__SHIFT) |
2932                                 (mpll_param.post_div << MPLL_AD_FUNC_CNTL__YCLK_POST_DIV__SHIFT);
2933         }
2934
2935         if (pi->caps_mclk_ss_support) {
2936                 struct amdgpu_atom_ss ss;
2937                 u32 freq_nom;
2938                 u32 tmp;
2939                 u32 reference_clock = adev->clock.mpll.reference_freq;
2940
2941                 if (mpll_param.qdr == 1)
2942                         freq_nom = memory_clock * 4 * (1 << mpll_param.post_div);
2943                 else
2944                         freq_nom = memory_clock * 2 * (1 << mpll_param.post_div);
2945
2946                 tmp = (freq_nom / reference_clock);
2947                 tmp = tmp * tmp;
2948                 if (amdgpu_atombios_get_asic_ss_info(adev, &ss,
2949                                                      ASIC_INTERNAL_MEMORY_SS, freq_nom)) {
2950                         u32 clks = reference_clock * 5 / ss.rate;
2951                         u32 clkv = (u32)((((131 * ss.percentage * ss.rate) / 100) * tmp) / freq_nom);
2952
2953                         mpll_ss1 &= ~MPLL_SS1__CLKV_MASK;
2954                         mpll_ss1 |= (clkv << MPLL_SS1__CLKV__SHIFT);
2955
2956                         mpll_ss2 &= ~MPLL_SS2__CLKS_MASK;
2957                         mpll_ss2 |= (clks << MPLL_SS2__CLKS__SHIFT);
2958                 }
2959         }
2960
2961         mclk_pwrmgt_cntl &= ~MCLK_PWRMGT_CNTL__DLL_SPEED_MASK;
2962         mclk_pwrmgt_cntl |= (mpll_param.dll_speed << MCLK_PWRMGT_CNTL__DLL_SPEED__SHIFT);
2963
2964         if (dll_state_on)
2965                 mclk_pwrmgt_cntl |= MCLK_PWRMGT_CNTL__MRDCK0_PDNB_MASK |
2966                         MCLK_PWRMGT_CNTL__MRDCK1_PDNB_MASK;
2967         else
2968                 mclk_pwrmgt_cntl &= ~(MCLK_PWRMGT_CNTL__MRDCK0_PDNB_MASK |
2969                         MCLK_PWRMGT_CNTL__MRDCK1_PDNB_MASK);
2970
2971         mclk->MclkFrequency = memory_clock;
2972         mclk->MpllFuncCntl = mpll_func_cntl;
2973         mclk->MpllFuncCntl_1 = mpll_func_cntl_1;
2974         mclk->MpllFuncCntl_2 = mpll_func_cntl_2;
2975         mclk->MpllAdFuncCntl = mpll_ad_func_cntl;
2976         mclk->MpllDqFuncCntl = mpll_dq_func_cntl;
2977         mclk->MclkPwrmgtCntl = mclk_pwrmgt_cntl;
2978         mclk->DllCntl = dll_cntl;
2979         mclk->MpllSs1 = mpll_ss1;
2980         mclk->MpllSs2 = mpll_ss2;
2981
2982         return 0;
2983 }
2984
2985 static int ci_populate_single_memory_level(struct amdgpu_device *adev,
2986                                            u32 memory_clock,
2987                                            SMU7_Discrete_MemoryLevel *memory_level)
2988 {
2989         struct ci_power_info *pi = ci_get_pi(adev);
2990         int ret;
2991         bool dll_state_on;
2992
2993         if (adev->pm.dpm.dyn_state.vddc_dependency_on_mclk.entries) {
2994                 ret = ci_get_dependency_volt_by_clk(adev,
2995                                                     &adev->pm.dpm.dyn_state.vddc_dependency_on_mclk,
2996                                                     memory_clock, &memory_level->MinVddc);
2997                 if (ret)
2998                         return ret;
2999         }
3000
3001         if (adev->pm.dpm.dyn_state.vddci_dependency_on_mclk.entries) {
3002                 ret = ci_get_dependency_volt_by_clk(adev,
3003                                                     &adev->pm.dpm.dyn_state.vddci_dependency_on_mclk,
3004                                                     memory_clock, &memory_level->MinVddci);
3005                 if (ret)
3006                         return ret;
3007         }
3008
3009         if (adev->pm.dpm.dyn_state.mvdd_dependency_on_mclk.entries) {
3010                 ret = ci_get_dependency_volt_by_clk(adev,
3011                                                     &adev->pm.dpm.dyn_state.mvdd_dependency_on_mclk,
3012                                                     memory_clock, &memory_level->MinMvdd);
3013                 if (ret)
3014                         return ret;
3015         }
3016
3017         memory_level->MinVddcPhases = 1;
3018
3019         if (pi->vddc_phase_shed_control)
3020                 ci_populate_phase_value_based_on_mclk(adev,
3021                                                       &adev->pm.dpm.dyn_state.phase_shedding_limits_table,
3022                                                       memory_clock,
3023                                                       &memory_level->MinVddcPhases);
3024
3025         memory_level->EnabledForThrottle = 1;
3026         memory_level->EnabledForActivity = 1;
3027         memory_level->UpH = 0;
3028         memory_level->DownH = 100;
3029         memory_level->VoltageDownH = 0;
3030         memory_level->ActivityLevel = (u16)pi->mclk_activity_target;
3031
3032         memory_level->StutterEnable = false;
3033         memory_level->StrobeEnable = false;
3034         memory_level->EdcReadEnable = false;
3035         memory_level->EdcWriteEnable = false;
3036         memory_level->RttEnable = false;
3037
3038         memory_level->DisplayWatermark = PPSMC_DISPLAY_WATERMARK_LOW;
3039
3040         if (pi->mclk_stutter_mode_threshold &&
3041             (memory_clock <= pi->mclk_stutter_mode_threshold) &&
3042             (pi->uvd_enabled == false) &&
3043             (RREG32(mmDPG_PIPE_STUTTER_CONTROL) & DPG_PIPE_STUTTER_CONTROL__STUTTER_ENABLE_MASK) &&
3044             (adev->pm.dpm.new_active_crtc_count <= 2))
3045                 memory_level->StutterEnable = true;
3046
3047         if (pi->mclk_strobe_mode_threshold &&
3048             (memory_clock <= pi->mclk_strobe_mode_threshold))
3049                 memory_level->StrobeEnable = 1;
3050
3051         if (adev->mc.vram_type == AMDGPU_VRAM_TYPE_GDDR5) {
3052                 memory_level->StrobeRatio =
3053                         ci_get_mclk_frequency_ratio(memory_clock, memory_level->StrobeEnable);
3054                 if (pi->mclk_edc_enable_threshold &&
3055                     (memory_clock > pi->mclk_edc_enable_threshold))
3056                         memory_level->EdcReadEnable = true;
3057
3058                 if (pi->mclk_edc_wr_enable_threshold &&
3059                     (memory_clock > pi->mclk_edc_wr_enable_threshold))
3060                         memory_level->EdcWriteEnable = true;
3061
3062                 if (memory_level->StrobeEnable) {
3063                         if (ci_get_mclk_frequency_ratio(memory_clock, true) >=
3064                             ((RREG32(mmMC_SEQ_MISC7) >> 16) & 0xf))
3065                                 dll_state_on = ((RREG32(mmMC_SEQ_MISC5) >> 1) & 0x1) ? true : false;
3066                         else
3067                                 dll_state_on = ((RREG32(mmMC_SEQ_MISC6) >> 1) & 0x1) ? true : false;
3068                 } else {
3069                         dll_state_on = pi->dll_default_on;
3070                 }
3071         } else {
3072                 memory_level->StrobeRatio = ci_get_ddr3_mclk_frequency_ratio(memory_clock);
3073                 dll_state_on = ((RREG32(mmMC_SEQ_MISC5) >> 1) & 0x1) ? true : false;
3074         }
3075
3076         ret = ci_calculate_mclk_params(adev, memory_clock, memory_level, memory_level->StrobeEnable, dll_state_on);
3077         if (ret)
3078                 return ret;
3079
3080         memory_level->MinVddc = cpu_to_be32(memory_level->MinVddc * VOLTAGE_SCALE);
3081         memory_level->MinVddcPhases = cpu_to_be32(memory_level->MinVddcPhases);
3082         memory_level->MinVddci = cpu_to_be32(memory_level->MinVddci * VOLTAGE_SCALE);
3083         memory_level->MinMvdd = cpu_to_be32(memory_level->MinMvdd * VOLTAGE_SCALE);
3084
3085         memory_level->MclkFrequency = cpu_to_be32(memory_level->MclkFrequency);
3086         memory_level->ActivityLevel = cpu_to_be16(memory_level->ActivityLevel);
3087         memory_level->MpllFuncCntl = cpu_to_be32(memory_level->MpllFuncCntl);
3088         memory_level->MpllFuncCntl_1 = cpu_to_be32(memory_level->MpllFuncCntl_1);
3089         memory_level->MpllFuncCntl_2 = cpu_to_be32(memory_level->MpllFuncCntl_2);
3090         memory_level->MpllAdFuncCntl = cpu_to_be32(memory_level->MpllAdFuncCntl);
3091         memory_level->MpllDqFuncCntl = cpu_to_be32(memory_level->MpllDqFuncCntl);
3092         memory_level->MclkPwrmgtCntl = cpu_to_be32(memory_level->MclkPwrmgtCntl);
3093         memory_level->DllCntl = cpu_to_be32(memory_level->DllCntl);
3094         memory_level->MpllSs1 = cpu_to_be32(memory_level->MpllSs1);
3095         memory_level->MpllSs2 = cpu_to_be32(memory_level->MpllSs2);
3096
3097         return 0;
3098 }
3099
3100 static int ci_populate_smc_acpi_level(struct amdgpu_device *adev,
3101                                       SMU7_Discrete_DpmTable *table)
3102 {
3103         struct ci_power_info *pi = ci_get_pi(adev);
3104         struct atom_clock_dividers dividers;
3105         SMU7_Discrete_VoltageLevel voltage_level;
3106         u32 spll_func_cntl = pi->clock_registers.cg_spll_func_cntl;
3107         u32 spll_func_cntl_2 = pi->clock_registers.cg_spll_func_cntl_2;
3108         u32 dll_cntl = pi->clock_registers.dll_cntl;
3109         u32 mclk_pwrmgt_cntl = pi->clock_registers.mclk_pwrmgt_cntl;
3110         int ret;
3111
3112         table->ACPILevel.Flags &= ~PPSMC_SWSTATE_FLAG_DC;
3113
3114         if (pi->acpi_vddc)
3115                 table->ACPILevel.MinVddc = cpu_to_be32(pi->acpi_vddc * VOLTAGE_SCALE);
3116         else
3117                 table->ACPILevel.MinVddc = cpu_to_be32(pi->min_vddc_in_pp_table * VOLTAGE_SCALE);
3118
3119         table->ACPILevel.MinVddcPhases = pi->vddc_phase_shed_control ? 0 : 1;
3120
3121         table->ACPILevel.SclkFrequency = adev->clock.spll.reference_freq;
3122
3123         ret = amdgpu_atombios_get_clock_dividers(adev,
3124                                                  COMPUTE_GPUCLK_INPUT_FLAG_SCLK,
3125                                                  table->ACPILevel.SclkFrequency, false, &dividers);
3126         if (ret)
3127                 return ret;
3128
3129         table->ACPILevel.SclkDid = (u8)dividers.post_divider;
3130         table->ACPILevel.DisplayWatermark = PPSMC_DISPLAY_WATERMARK_LOW;
3131         table->ACPILevel.DeepSleepDivId = 0;
3132
3133         spll_func_cntl &= ~CG_SPLL_FUNC_CNTL__SPLL_PWRON_MASK;
3134         spll_func_cntl |= CG_SPLL_FUNC_CNTL__SPLL_RESET_MASK;
3135
3136         spll_func_cntl_2 &= ~CG_SPLL_FUNC_CNTL_2__SCLK_MUX_SEL_MASK;
3137         spll_func_cntl_2 |= (4 << CG_SPLL_FUNC_CNTL_2__SCLK_MUX_SEL__SHIFT);
3138
3139         table->ACPILevel.CgSpllFuncCntl = spll_func_cntl;
3140         table->ACPILevel.CgSpllFuncCntl2 = spll_func_cntl_2;
3141         table->ACPILevel.CgSpllFuncCntl3 = pi->clock_registers.cg_spll_func_cntl_3;
3142         table->ACPILevel.CgSpllFuncCntl4 = pi->clock_registers.cg_spll_func_cntl_4;
3143         table->ACPILevel.SpllSpreadSpectrum = pi->clock_registers.cg_spll_spread_spectrum;
3144         table->ACPILevel.SpllSpreadSpectrum2 = pi->clock_registers.cg_spll_spread_spectrum_2;
3145         table->ACPILevel.CcPwrDynRm = 0;
3146         table->ACPILevel.CcPwrDynRm1 = 0;
3147
3148         table->ACPILevel.Flags = cpu_to_be32(table->ACPILevel.Flags);
3149         table->ACPILevel.MinVddcPhases = cpu_to_be32(table->ACPILevel.MinVddcPhases);
3150         table->ACPILevel.SclkFrequency = cpu_to_be32(table->ACPILevel.SclkFrequency);
3151         table->ACPILevel.CgSpllFuncCntl = cpu_to_be32(table->ACPILevel.CgSpllFuncCntl);
3152         table->ACPILevel.CgSpllFuncCntl2 = cpu_to_be32(table->ACPILevel.CgSpllFuncCntl2);
3153         table->ACPILevel.CgSpllFuncCntl3 = cpu_to_be32(table->ACPILevel.CgSpllFuncCntl3);
3154         table->ACPILevel.CgSpllFuncCntl4 = cpu_to_be32(table->ACPILevel.CgSpllFuncCntl4);
3155         table->ACPILevel.SpllSpreadSpectrum = cpu_to_be32(table->ACPILevel.SpllSpreadSpectrum);
3156         table->ACPILevel.SpllSpreadSpectrum2 = cpu_to_be32(table->ACPILevel.SpllSpreadSpectrum2);
3157         table->ACPILevel.CcPwrDynRm = cpu_to_be32(table->ACPILevel.CcPwrDynRm);
3158         table->ACPILevel.CcPwrDynRm1 = cpu_to_be32(table->ACPILevel.CcPwrDynRm1);
3159
3160         table->MemoryACPILevel.MinVddc = table->ACPILevel.MinVddc;
3161         table->MemoryACPILevel.MinVddcPhases = table->ACPILevel.MinVddcPhases;
3162
3163         if (pi->vddci_control != CISLANDS_VOLTAGE_CONTROL_NONE) {
3164                 if (pi->acpi_vddci)
3165                         table->MemoryACPILevel.MinVddci =
3166                                 cpu_to_be32(pi->acpi_vddci * VOLTAGE_SCALE);
3167                 else
3168                         table->MemoryACPILevel.MinVddci =
3169                                 cpu_to_be32(pi->min_vddci_in_pp_table * VOLTAGE_SCALE);
3170         }
3171
3172         if (ci_populate_mvdd_value(adev, 0, &voltage_level))
3173                 table->MemoryACPILevel.MinMvdd = 0;
3174         else
3175                 table->MemoryACPILevel.MinMvdd =
3176                         cpu_to_be32(voltage_level.Voltage * VOLTAGE_SCALE);
3177
3178         mclk_pwrmgt_cntl |= MCLK_PWRMGT_CNTL__MRDCK0_RESET_MASK |
3179                 MCLK_PWRMGT_CNTL__MRDCK1_RESET_MASK;
3180         mclk_pwrmgt_cntl &= ~(MCLK_PWRMGT_CNTL__MRDCK0_PDNB_MASK |
3181                         MCLK_PWRMGT_CNTL__MRDCK1_PDNB_MASK);
3182
3183         dll_cntl &= ~(DLL_CNTL__MRDCK0_BYPASS_MASK | DLL_CNTL__MRDCK1_BYPASS_MASK);
3184
3185         table->MemoryACPILevel.DllCntl = cpu_to_be32(dll_cntl);
3186         table->MemoryACPILevel.MclkPwrmgtCntl = cpu_to_be32(mclk_pwrmgt_cntl);
3187         table->MemoryACPILevel.MpllAdFuncCntl =
3188                 cpu_to_be32(pi->clock_registers.mpll_ad_func_cntl);
3189         table->MemoryACPILevel.MpllDqFuncCntl =
3190                 cpu_to_be32(pi->clock_registers.mpll_dq_func_cntl);
3191         table->MemoryACPILevel.MpllFuncCntl =
3192                 cpu_to_be32(pi->clock_registers.mpll_func_cntl);
3193         table->MemoryACPILevel.MpllFuncCntl_1 =
3194                 cpu_to_be32(pi->clock_registers.mpll_func_cntl_1);
3195         table->MemoryACPILevel.MpllFuncCntl_2 =
3196                 cpu_to_be32(pi->clock_registers.mpll_func_cntl_2);
3197         table->MemoryACPILevel.MpllSs1 = cpu_to_be32(pi->clock_registers.mpll_ss1);
3198         table->MemoryACPILevel.MpllSs2 = cpu_to_be32(pi->clock_registers.mpll_ss2);
3199
3200         table->MemoryACPILevel.EnabledForThrottle = 0;
3201         table->MemoryACPILevel.EnabledForActivity = 0;
3202         table->MemoryACPILevel.UpH = 0;
3203         table->MemoryACPILevel.DownH = 100;
3204         table->MemoryACPILevel.VoltageDownH = 0;
3205         table->MemoryACPILevel.ActivityLevel =
3206                 cpu_to_be16((u16)pi->mclk_activity_target);
3207
3208         table->MemoryACPILevel.StutterEnable = false;
3209         table->MemoryACPILevel.StrobeEnable = false;
3210         table->MemoryACPILevel.EdcReadEnable = false;
3211         table->MemoryACPILevel.EdcWriteEnable = false;
3212         table->MemoryACPILevel.RttEnable = false;
3213
3214         return 0;
3215 }
3216
3217
3218 static int ci_enable_ulv(struct amdgpu_device *adev, bool enable)
3219 {
3220         struct ci_power_info *pi = ci_get_pi(adev);
3221         struct ci_ulv_parm *ulv = &pi->ulv;
3222
3223         if (ulv->supported) {
3224                 if (enable)
3225                         return (amdgpu_ci_send_msg_to_smc(adev, PPSMC_MSG_EnableULV) == PPSMC_Result_OK) ?
3226                                 0 : -EINVAL;
3227                 else
3228                         return (amdgpu_ci_send_msg_to_smc(adev, PPSMC_MSG_DisableULV) == PPSMC_Result_OK) ?
3229                                 0 : -EINVAL;
3230         }
3231
3232         return 0;
3233 }
3234
3235 static int ci_populate_ulv_level(struct amdgpu_device *adev,
3236                                  SMU7_Discrete_Ulv *state)
3237 {
3238         struct ci_power_info *pi = ci_get_pi(adev);
3239         u16 ulv_voltage = adev->pm.dpm.backbias_response_time;
3240
3241         state->CcPwrDynRm = 0;
3242         state->CcPwrDynRm1 = 0;
3243
3244         if (ulv_voltage == 0) {
3245                 pi->ulv.supported = false;
3246                 return 0;
3247         }
3248
3249         if (pi->voltage_control != CISLANDS_VOLTAGE_CONTROL_BY_SVID2) {
3250                 if (ulv_voltage > adev->pm.dpm.dyn_state.vddc_dependency_on_sclk.entries[0].v)
3251                         state->VddcOffset = 0;
3252                 else
3253                         state->VddcOffset =
3254                                 adev->pm.dpm.dyn_state.vddc_dependency_on_sclk.entries[0].v - ulv_voltage;
3255         } else {
3256                 if (ulv_voltage > adev->pm.dpm.dyn_state.vddc_dependency_on_sclk.entries[0].v)
3257                         state->VddcOffsetVid = 0;
3258                 else
3259                         state->VddcOffsetVid = (u8)
3260                                 ((adev->pm.dpm.dyn_state.vddc_dependency_on_sclk.entries[0].v - ulv_voltage) *
3261                                  VOLTAGE_VID_OFFSET_SCALE2 / VOLTAGE_VID_OFFSET_SCALE1);
3262         }
3263         state->VddcPhase = pi->vddc_phase_shed_control ? 0 : 1;
3264
3265         state->CcPwrDynRm = cpu_to_be32(state->CcPwrDynRm);
3266         state->CcPwrDynRm1 = cpu_to_be32(state->CcPwrDynRm1);
3267         state->VddcOffset = cpu_to_be16(state->VddcOffset);
3268
3269         return 0;
3270 }
3271
3272 static int ci_calculate_sclk_params(struct amdgpu_device *adev,
3273                                     u32 engine_clock,
3274                                     SMU7_Discrete_GraphicsLevel *sclk)
3275 {
3276         struct ci_power_info *pi = ci_get_pi(adev);
3277         struct atom_clock_dividers dividers;
3278         u32 spll_func_cntl_3 = pi->clock_registers.cg_spll_func_cntl_3;
3279         u32 spll_func_cntl_4 = pi->clock_registers.cg_spll_func_cntl_4;
3280         u32 cg_spll_spread_spectrum = pi->clock_registers.cg_spll_spread_spectrum;
3281         u32 cg_spll_spread_spectrum_2 = pi->clock_registers.cg_spll_spread_spectrum_2;
3282         u32 reference_clock = adev->clock.spll.reference_freq;
3283         u32 reference_divider;
3284         u32 fbdiv;
3285         int ret;
3286
3287         ret = amdgpu_atombios_get_clock_dividers(adev,
3288                                                  COMPUTE_GPUCLK_INPUT_FLAG_SCLK,
3289                                                  engine_clock, false, &dividers);
3290         if (ret)
3291                 return ret;
3292
3293         reference_divider = 1 + dividers.ref_div;
3294         fbdiv = dividers.fb_div & 0x3FFFFFF;
3295
3296         spll_func_cntl_3 &= ~CG_SPLL_FUNC_CNTL_3__SPLL_FB_DIV_MASK;
3297         spll_func_cntl_3 |= (fbdiv << CG_SPLL_FUNC_CNTL_3__SPLL_FB_DIV__SHIFT);
3298         spll_func_cntl_3 |= CG_SPLL_FUNC_CNTL_3__SPLL_DITHEN_MASK;
3299
3300         if (pi->caps_sclk_ss_support) {
3301                 struct amdgpu_atom_ss ss;
3302                 u32 vco_freq = engine_clock * dividers.post_div;
3303
3304                 if (amdgpu_atombios_get_asic_ss_info(adev, &ss,
3305                                                      ASIC_INTERNAL_ENGINE_SS, vco_freq)) {
3306                         u32 clk_s = reference_clock * 5 / (reference_divider * ss.rate);
3307                         u32 clk_v = 4 * ss.percentage * fbdiv / (clk_s * 10000);
3308
3309                         cg_spll_spread_spectrum &= ~(CG_SPLL_SPREAD_SPECTRUM__CLKS_MASK | CG_SPLL_SPREAD_SPECTRUM__SSEN_MASK);
3310                         cg_spll_spread_spectrum |= (clk_s << CG_SPLL_SPREAD_SPECTRUM__CLKS__SHIFT);
3311                         cg_spll_spread_spectrum |= (1 << CG_SPLL_SPREAD_SPECTRUM__SSEN__SHIFT);
3312
3313                         cg_spll_spread_spectrum_2 &= ~CG_SPLL_SPREAD_SPECTRUM_2__CLKV_MASK;
3314                         cg_spll_spread_spectrum_2 |= (clk_v << CG_SPLL_SPREAD_SPECTRUM_2__CLKV__SHIFT);
3315                 }
3316         }
3317
3318         sclk->SclkFrequency = engine_clock;
3319         sclk->CgSpllFuncCntl3 = spll_func_cntl_3;
3320         sclk->CgSpllFuncCntl4 = spll_func_cntl_4;
3321         sclk->SpllSpreadSpectrum = cg_spll_spread_spectrum;
3322         sclk->SpllSpreadSpectrum2  = cg_spll_spread_spectrum_2;
3323         sclk->SclkDid = (u8)dividers.post_divider;
3324
3325         return 0;
3326 }
3327
3328 static int ci_populate_single_graphic_level(struct amdgpu_device *adev,
3329                                             u32 engine_clock,
3330                                             u16 sclk_activity_level_t,
3331                                             SMU7_Discrete_GraphicsLevel *graphic_level)
3332 {
3333         struct ci_power_info *pi = ci_get_pi(adev);
3334         int ret;
3335
3336         ret = ci_calculate_sclk_params(adev, engine_clock, graphic_level);
3337         if (ret)
3338                 return ret;
3339
3340         ret = ci_get_dependency_volt_by_clk(adev,
3341                                             &adev->pm.dpm.dyn_state.vddc_dependency_on_sclk,
3342                                             engine_clock, &graphic_level->MinVddc);
3343         if (ret)
3344                 return ret;
3345
3346         graphic_level->SclkFrequency = engine_clock;
3347
3348         graphic_level->Flags =  0;
3349         graphic_level->MinVddcPhases = 1;
3350
3351         if (pi->vddc_phase_shed_control)
3352                 ci_populate_phase_value_based_on_sclk(adev,
3353                                                       &adev->pm.dpm.dyn_state.phase_shedding_limits_table,
3354                                                       engine_clock,
3355                                                       &graphic_level->MinVddcPhases);
3356
3357         graphic_level->ActivityLevel = sclk_activity_level_t;
3358
3359         graphic_level->CcPwrDynRm = 0;
3360         graphic_level->CcPwrDynRm1 = 0;
3361         graphic_level->EnabledForThrottle = 1;
3362         graphic_level->UpH = 0;
3363         graphic_level->DownH = 0;
3364         graphic_level->VoltageDownH = 0;
3365         graphic_level->PowerThrottle = 0;
3366
3367         if (pi->caps_sclk_ds)
3368                 graphic_level->DeepSleepDivId = ci_get_sleep_divider_id_from_clock(adev,
3369                                                                                    engine_clock,
3370                                                                                    CISLAND_MINIMUM_ENGINE_CLOCK);
3371
3372         graphic_level->DisplayWatermark = PPSMC_DISPLAY_WATERMARK_LOW;
3373
3374         graphic_level->Flags = cpu_to_be32(graphic_level->Flags);
3375         graphic_level->MinVddc = cpu_to_be32(graphic_level->MinVddc * VOLTAGE_SCALE);
3376         graphic_level->MinVddcPhases = cpu_to_be32(graphic_level->MinVddcPhases);
3377         graphic_level->SclkFrequency = cpu_to_be32(graphic_level->SclkFrequency);
3378         graphic_level->ActivityLevel = cpu_to_be16(graphic_level->ActivityLevel);
3379         graphic_level->CgSpllFuncCntl3 = cpu_to_be32(graphic_level->CgSpllFuncCntl3);
3380         graphic_level->CgSpllFuncCntl4 = cpu_to_be32(graphic_level->CgSpllFuncCntl4);
3381         graphic_level->SpllSpreadSpectrum = cpu_to_be32(graphic_level->SpllSpreadSpectrum);
3382         graphic_level->SpllSpreadSpectrum2 = cpu_to_be32(graphic_level->SpllSpreadSpectrum2);
3383         graphic_level->CcPwrDynRm = cpu_to_be32(graphic_level->CcPwrDynRm);
3384         graphic_level->CcPwrDynRm1 = cpu_to_be32(graphic_level->CcPwrDynRm1);
3385         graphic_level->EnabledForActivity = 1;
3386
3387         return 0;
3388 }
3389
3390 static int ci_populate_all_graphic_levels(struct amdgpu_device *adev)
3391 {
3392         struct ci_power_info *pi = ci_get_pi(adev);
3393         struct ci_dpm_table *dpm_table = &pi->dpm_table;
3394         u32 level_array_address = pi->dpm_table_start +
3395                 offsetof(SMU7_Discrete_DpmTable, GraphicsLevel);
3396         u32 level_array_size = sizeof(SMU7_Discrete_GraphicsLevel) *
3397                 SMU7_MAX_LEVELS_GRAPHICS;
3398         SMU7_Discrete_GraphicsLevel *levels = pi->smc_state_table.GraphicsLevel;
3399         u32 i, ret;
3400
3401         memset(levels, 0, level_array_size);
3402
3403         for (i = 0; i < dpm_table->sclk_table.count; i++) {
3404                 ret = ci_populate_single_graphic_level(adev,
3405                                                        dpm_table->sclk_table.dpm_levels[i].value,
3406                                                        (u16)pi->activity_target[i],
3407                                                        &pi->smc_state_table.GraphicsLevel[i]);
3408                 if (ret)
3409                         return ret;
3410                 if (i > 1)
3411                         pi->smc_state_table.GraphicsLevel[i].DeepSleepDivId = 0;
3412                 if (i == (dpm_table->sclk_table.count - 1))
3413                         pi->smc_state_table.GraphicsLevel[i].DisplayWatermark =
3414                                 PPSMC_DISPLAY_WATERMARK_HIGH;
3415         }
3416
3417         pi->smc_state_table.GraphicsDpmLevelCount = (u8)dpm_table->sclk_table.count;
3418         pi->dpm_level_enable_mask.sclk_dpm_enable_mask =
3419                 ci_get_dpm_level_enable_mask_value(&dpm_table->sclk_table);
3420
3421         ret = amdgpu_ci_copy_bytes_to_smc(adev, level_array_address,
3422                                    (u8 *)levels, level_array_size,
3423                                    pi->sram_end);
3424         if (ret)
3425                 return ret;
3426
3427         return 0;
3428 }
3429
3430 static int ci_populate_ulv_state(struct amdgpu_device *adev,
3431                                  SMU7_Discrete_Ulv *ulv_level)
3432 {
3433         return ci_populate_ulv_level(adev, ulv_level);
3434 }
3435
3436 static int ci_populate_all_memory_levels(struct amdgpu_device *adev)
3437 {
3438         struct ci_power_info *pi = ci_get_pi(adev);
3439         struct ci_dpm_table *dpm_table = &pi->dpm_table;
3440         u32 level_array_address = pi->dpm_table_start +
3441                 offsetof(SMU7_Discrete_DpmTable, MemoryLevel);
3442         u32 level_array_size = sizeof(SMU7_Discrete_MemoryLevel) *
3443                 SMU7_MAX_LEVELS_MEMORY;
3444         SMU7_Discrete_MemoryLevel *levels = pi->smc_state_table.MemoryLevel;
3445         u32 i, ret;
3446
3447         memset(levels, 0, level_array_size);
3448
3449         for (i = 0; i < dpm_table->mclk_table.count; i++) {
3450                 if (dpm_table->mclk_table.dpm_levels[i].value == 0)
3451                         return -EINVAL;
3452                 ret = ci_populate_single_memory_level(adev,
3453                                                       dpm_table->mclk_table.dpm_levels[i].value,
3454                                                       &pi->smc_state_table.MemoryLevel[i]);
3455                 if (ret)
3456                         return ret;
3457         }
3458
3459         if ((dpm_table->mclk_table.count >= 2) &&
3460             ((adev->pdev->device == 0x67B0) || (adev->pdev->device == 0x67B1))) {
3461                 pi->smc_state_table.MemoryLevel[1].MinVddc =
3462                         pi->smc_state_table.MemoryLevel[0].MinVddc;
3463                 pi->smc_state_table.MemoryLevel[1].MinVddcPhases =
3464                         pi->smc_state_table.MemoryLevel[0].MinVddcPhases;
3465         }
3466
3467         pi->smc_state_table.MemoryLevel[0].ActivityLevel = cpu_to_be16(0x1F);
3468
3469         pi->smc_state_table.MemoryDpmLevelCount = (u8)dpm_table->mclk_table.count;
3470         pi->dpm_level_enable_mask.mclk_dpm_enable_mask =
3471                 ci_get_dpm_level_enable_mask_value(&dpm_table->mclk_table);
3472
3473         pi->smc_state_table.MemoryLevel[dpm_table->mclk_table.count - 1].DisplayWatermark =
3474                 PPSMC_DISPLAY_WATERMARK_HIGH;
3475
3476         ret = amdgpu_ci_copy_bytes_to_smc(adev, level_array_address,
3477                                    (u8 *)levels, level_array_size,
3478                                    pi->sram_end);
3479         if (ret)
3480                 return ret;
3481
3482         return 0;
3483 }
3484
3485 static void ci_reset_single_dpm_table(struct amdgpu_device *adev,
3486                                       struct ci_single_dpm_table* dpm_table,
3487                                       u32 count)
3488 {
3489         u32 i;
3490
3491         dpm_table->count = count;
3492         for (i = 0; i < MAX_REGULAR_DPM_NUMBER; i++)
3493                 dpm_table->dpm_levels[i].enabled = false;
3494 }
3495
3496 static void ci_setup_pcie_table_entry(struct ci_single_dpm_table* dpm_table,
3497                                       u32 index, u32 pcie_gen, u32 pcie_lanes)
3498 {
3499         dpm_table->dpm_levels[index].value = pcie_gen;
3500         dpm_table->dpm_levels[index].param1 = pcie_lanes;
3501         dpm_table->dpm_levels[index].enabled = true;
3502 }
3503
3504 static int ci_setup_default_pcie_tables(struct amdgpu_device *adev)
3505 {
3506         struct ci_power_info *pi = ci_get_pi(adev);
3507
3508         if (!pi->use_pcie_performance_levels && !pi->use_pcie_powersaving_levels)
3509                 return -EINVAL;
3510
3511         if (pi->use_pcie_performance_levels && !pi->use_pcie_powersaving_levels) {
3512                 pi->pcie_gen_powersaving = pi->pcie_gen_performance;
3513                 pi->pcie_lane_powersaving = pi->pcie_lane_performance;
3514         } else if (!pi->use_pcie_performance_levels && pi->use_pcie_powersaving_levels) {
3515                 pi->pcie_gen_performance = pi->pcie_gen_powersaving;
3516                 pi->pcie_lane_performance = pi->pcie_lane_powersaving;
3517         }
3518
3519         ci_reset_single_dpm_table(adev,
3520                                   &pi->dpm_table.pcie_speed_table,
3521                                   SMU7_MAX_LEVELS_LINK);
3522
3523         if (adev->asic_type == CHIP_BONAIRE)
3524                 ci_setup_pcie_table_entry(&pi->dpm_table.pcie_speed_table, 0,
3525                                           pi->pcie_gen_powersaving.min,
3526                                           pi->pcie_lane_powersaving.max);
3527         else
3528                 ci_setup_pcie_table_entry(&pi->dpm_table.pcie_speed_table, 0,
3529                                           pi->pcie_gen_powersaving.min,
3530                                           pi->pcie_lane_powersaving.min);
3531         ci_setup_pcie_table_entry(&pi->dpm_table.pcie_speed_table, 1,
3532                                   pi->pcie_gen_performance.min,
3533                                   pi->pcie_lane_performance.min);
3534         ci_setup_pcie_table_entry(&pi->dpm_table.pcie_speed_table, 2,
3535                                   pi->pcie_gen_powersaving.min,
3536                                   pi->pcie_lane_powersaving.max);
3537         ci_setup_pcie_table_entry(&pi->dpm_table.pcie_speed_table, 3,
3538                                   pi->pcie_gen_performance.min,
3539                                   pi->pcie_lane_performance.max);
3540         ci_setup_pcie_table_entry(&pi->dpm_table.pcie_speed_table, 4,
3541                                   pi->pcie_gen_powersaving.max,
3542                                   pi->pcie_lane_powersaving.max);
3543         ci_setup_pcie_table_entry(&pi->dpm_table.pcie_speed_table, 5,
3544                                   pi->pcie_gen_performance.max,
3545                                   pi->pcie_lane_performance.max);
3546
3547         pi->dpm_table.pcie_speed_table.count = 6;
3548
3549         return 0;
3550 }
3551
3552 static int ci_setup_default_dpm_tables(struct amdgpu_device *adev)
3553 {
3554         struct ci_power_info *pi = ci_get_pi(adev);
3555         struct amdgpu_clock_voltage_dependency_table *allowed_sclk_vddc_table =
3556                 &adev->pm.dpm.dyn_state.vddc_dependency_on_sclk;
3557         struct amdgpu_clock_voltage_dependency_table *allowed_mclk_table =
3558                 &adev->pm.dpm.dyn_state.vddc_dependency_on_mclk;
3559         struct amdgpu_cac_leakage_table *std_voltage_table =
3560                 &adev->pm.dpm.dyn_state.cac_leakage_table;
3561         u32 i;
3562
3563         if (allowed_sclk_vddc_table == NULL)
3564                 return -EINVAL;
3565         if (allowed_sclk_vddc_table->count < 1)
3566                 return -EINVAL;
3567         if (allowed_mclk_table == NULL)
3568                 return -EINVAL;
3569         if (allowed_mclk_table->count < 1)
3570                 return -EINVAL;
3571
3572         memset(&pi->dpm_table, 0, sizeof(struct ci_dpm_table));
3573
3574         ci_reset_single_dpm_table(adev,
3575                                   &pi->dpm_table.sclk_table,
3576                                   SMU7_MAX_LEVELS_GRAPHICS);
3577         ci_reset_single_dpm_table(adev,
3578                                   &pi->dpm_table.mclk_table,
3579                                   SMU7_MAX_LEVELS_MEMORY);
3580         ci_reset_single_dpm_table(adev,
3581                                   &pi->dpm_table.vddc_table,
3582                                   SMU7_MAX_LEVELS_VDDC);
3583         ci_reset_single_dpm_table(adev,
3584                                   &pi->dpm_table.vddci_table,
3585                                   SMU7_MAX_LEVELS_VDDCI);
3586         ci_reset_single_dpm_table(adev,
3587                                   &pi->dpm_table.mvdd_table,
3588                                   SMU7_MAX_LEVELS_MVDD);
3589
3590         pi->dpm_table.sclk_table.count = 0;
3591         for (i = 0; i < allowed_sclk_vddc_table->count; i++) {
3592                 if ((i == 0) ||
3593                     (pi->dpm_table.sclk_table.dpm_levels[pi->dpm_table.sclk_table.count-1].value !=
3594                      allowed_sclk_vddc_table->entries[i].clk)) {
3595                         pi->dpm_table.sclk_table.dpm_levels[pi->dpm_table.sclk_table.count].value =
3596                                 allowed_sclk_vddc_table->entries[i].clk;
3597                         pi->dpm_table.sclk_table.dpm_levels[pi->dpm_table.sclk_table.count].enabled =
3598                                 (i == 0) ? true : false;
3599                         pi->dpm_table.sclk_table.count++;
3600                 }
3601         }
3602
3603         pi->dpm_table.mclk_table.count = 0;
3604         for (i = 0; i < allowed_mclk_table->count; i++) {
3605                 if ((i == 0) ||
3606                     (pi->dpm_table.mclk_table.dpm_levels[pi->dpm_table.mclk_table.count-1].value !=
3607                      allowed_mclk_table->entries[i].clk)) {
3608                         pi->dpm_table.mclk_table.dpm_levels[pi->dpm_table.mclk_table.count].value =
3609                                 allowed_mclk_table->entries[i].clk;
3610                         pi->dpm_table.mclk_table.dpm_levels[pi->dpm_table.mclk_table.count].enabled =
3611                                 (i == 0) ? true : false;
3612                         pi->dpm_table.mclk_table.count++;
3613                 }
3614         }
3615
3616         for (i = 0; i < allowed_sclk_vddc_table->count; i++) {
3617                 pi->dpm_table.vddc_table.dpm_levels[i].value =
3618                         allowed_sclk_vddc_table->entries[i].v;
3619                 pi->dpm_table.vddc_table.dpm_levels[i].param1 =
3620                         std_voltage_table->entries[i].leakage;
3621                 pi->dpm_table.vddc_table.dpm_levels[i].enabled = true;
3622         }
3623         pi->dpm_table.vddc_table.count = allowed_sclk_vddc_table->count;
3624
3625         allowed_mclk_table = &adev->pm.dpm.dyn_state.vddci_dependency_on_mclk;
3626         if (allowed_mclk_table) {
3627                 for (i = 0; i < allowed_mclk_table->count; i++) {
3628                         pi->dpm_table.vddci_table.dpm_levels[i].value =
3629                                 allowed_mclk_table->entries[i].v;
3630                         pi->dpm_table.vddci_table.dpm_levels[i].enabled = true;
3631                 }
3632                 pi->dpm_table.vddci_table.count = allowed_mclk_table->count;
3633         }
3634
3635         allowed_mclk_table = &adev->pm.dpm.dyn_state.mvdd_dependency_on_mclk;
3636         if (allowed_mclk_table) {
3637                 for (i = 0; i < allowed_mclk_table->count; i++) {
3638                         pi->dpm_table.mvdd_table.dpm_levels[i].value =
3639                                 allowed_mclk_table->entries[i].v;
3640                         pi->dpm_table.mvdd_table.dpm_levels[i].enabled = true;
3641                 }
3642                 pi->dpm_table.mvdd_table.count = allowed_mclk_table->count;
3643         }
3644
3645         ci_setup_default_pcie_tables(adev);
3646
3647         return 0;
3648 }
3649
3650 static int ci_find_boot_level(struct ci_single_dpm_table *table,
3651                               u32 value, u32 *boot_level)
3652 {
3653         u32 i;
3654         int ret = -EINVAL;
3655
3656         for(i = 0; i < table->count; i++) {
3657                 if (value == table->dpm_levels[i].value) {
3658                         *boot_level = i;
3659                         ret = 0;
3660                 }
3661         }
3662
3663         return ret;
3664 }
3665
3666 static int ci_init_smc_table(struct amdgpu_device *adev)
3667 {
3668         struct ci_power_info *pi = ci_get_pi(adev);
3669         struct ci_ulv_parm *ulv = &pi->ulv;
3670         struct amdgpu_ps *amdgpu_boot_state = adev->pm.dpm.boot_ps;
3671         SMU7_Discrete_DpmTable *table = &pi->smc_state_table;
3672         int ret;
3673
3674         ret = ci_setup_default_dpm_tables(adev);
3675         if (ret)
3676                 return ret;
3677
3678         if (pi->voltage_control != CISLANDS_VOLTAGE_CONTROL_NONE)
3679                 ci_populate_smc_voltage_tables(adev, table);
3680
3681         ci_init_fps_limits(adev);
3682
3683         if (adev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_HARDWAREDC)
3684                 table->SystemFlags |= PPSMC_SYSTEMFLAG_GPIO_DC;
3685
3686         if (adev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_STEPVDDC)
3687                 table->SystemFlags |= PPSMC_SYSTEMFLAG_STEPVDDC;
3688
3689         if (adev->mc.vram_type == AMDGPU_VRAM_TYPE_GDDR5)
3690                 table->SystemFlags |= PPSMC_SYSTEMFLAG_GDDR5;
3691
3692         if (ulv->supported) {
3693                 ret = ci_populate_ulv_state(adev, &pi->smc_state_table.Ulv);
3694                 if (ret)
3695                         return ret;
3696                 WREG32_SMC(ixCG_ULV_PARAMETER, ulv->cg_ulv_parameter);
3697         }
3698
3699         ret = ci_populate_all_graphic_levels(adev);
3700         if (ret)
3701                 return ret;
3702
3703         ret = ci_populate_all_memory_levels(adev);
3704         if (ret)
3705                 return ret;
3706
3707         ci_populate_smc_link_level(adev, table);
3708
3709         ret = ci_populate_smc_acpi_level(adev, table);
3710         if (ret)
3711                 return ret;
3712
3713         ret = ci_populate_smc_vce_level(adev, table);
3714         if (ret)
3715                 return ret;
3716
3717         ret = ci_populate_smc_acp_level(adev, table);
3718         if (ret)
3719                 return ret;
3720
3721         ret = ci_populate_smc_samu_level(adev, table);
3722         if (ret)
3723                 return ret;
3724
3725         ret = ci_do_program_memory_timing_parameters(adev);
3726         if (ret)
3727                 return ret;
3728
3729         ret = ci_populate_smc_uvd_level(adev, table);
3730         if (ret)
3731                 return ret;
3732
3733         table->UvdBootLevel  = 0;
3734         table->VceBootLevel  = 0;
3735         table->AcpBootLevel  = 0;
3736         table->SamuBootLevel  = 0;
3737         table->GraphicsBootLevel  = 0;
3738         table->MemoryBootLevel  = 0;
3739
3740         ret = ci_find_boot_level(&pi->dpm_table.sclk_table,
3741                                  pi->vbios_boot_state.sclk_bootup_value,
3742                                  (u32 *)&pi->smc_state_table.GraphicsBootLevel);
3743
3744         ret = ci_find_boot_level(&pi->dpm_table.mclk_table,
3745                                  pi->vbios_boot_state.mclk_bootup_value,
3746                                  (u32 *)&pi->smc_state_table.MemoryBootLevel);
3747
3748         table->BootVddc = pi->vbios_boot_state.vddc_bootup_value;
3749         table->BootVddci = pi->vbios_boot_state.vddci_bootup_value;
3750         table->BootMVdd = pi->vbios_boot_state.mvdd_bootup_value;
3751
3752         ci_populate_smc_initial_state(adev, amdgpu_boot_state);
3753
3754         ret = ci_populate_bapm_parameters_in_dpm_table(adev);
3755         if (ret)
3756                 return ret;
3757
3758         table->UVDInterval = 1;
3759         table->VCEInterval = 1;
3760         table->ACPInterval = 1;
3761         table->SAMUInterval = 1;
3762         table->GraphicsVoltageChangeEnable = 1;
3763         table->GraphicsThermThrottleEnable = 1;
3764         table->GraphicsInterval = 1;
3765         table->VoltageInterval = 1;
3766         table->ThermalInterval = 1;
3767         table->TemperatureLimitHigh = (u16)((pi->thermal_temp_setting.temperature_high *
3768                                              CISLANDS_Q88_FORMAT_CONVERSION_UNIT) / 1000);
3769         table->TemperatureLimitLow = (u16)((pi->thermal_temp_setting.temperature_low *
3770                                             CISLANDS_Q88_FORMAT_CONVERSION_UNIT) / 1000);
3771         table->MemoryVoltageChangeEnable = 1;
3772         table->MemoryInterval = 1;
3773         table->VoltageResponseTime = 0;
3774         table->VddcVddciDelta = 4000;
3775         table->PhaseResponseTime = 0;
3776         table->MemoryThermThrottleEnable = 1;
3777         table->PCIeBootLinkLevel = pi->dpm_table.pcie_speed_table.count - 1;
3778         table->PCIeGenInterval = 1;
3779         if (pi->voltage_control == CISLANDS_VOLTAGE_CONTROL_BY_SVID2)
3780                 table->SVI2Enable  = 1;
3781         else
3782                 table->SVI2Enable  = 0;
3783
3784         table->ThermGpio = 17;
3785         table->SclkStepSize = 0x4000;
3786
3787         table->SystemFlags = cpu_to_be32(table->SystemFlags);
3788         table->SmioMaskVddcVid = cpu_to_be32(table->SmioMaskVddcVid);
3789         table->SmioMaskVddcPhase = cpu_to_be32(table->SmioMaskVddcPhase);
3790         table->SmioMaskVddciVid = cpu_to_be32(table->SmioMaskVddciVid);
3791         table->SmioMaskMvddVid = cpu_to_be32(table->SmioMaskMvddVid);
3792         table->SclkStepSize = cpu_to_be32(table->SclkStepSize);
3793         table->TemperatureLimitHigh = cpu_to_be16(table->TemperatureLimitHigh);
3794         table->TemperatureLimitLow = cpu_to_be16(table->TemperatureLimitLow);
3795         table->VddcVddciDelta = cpu_to_be16(table->VddcVddciDelta);
3796         table->VoltageResponseTime = cpu_to_be16(table->VoltageResponseTime);
3797         table->PhaseResponseTime = cpu_to_be16(table->PhaseResponseTime);
3798         table->BootVddc = cpu_to_be16(table->BootVddc * VOLTAGE_SCALE);
3799         table->BootVddci = cpu_to_be16(table->BootVddci * VOLTAGE_SCALE);
3800         table->BootMVdd = cpu_to_be16(table->BootMVdd * VOLTAGE_SCALE);
3801
3802         ret = amdgpu_ci_copy_bytes_to_smc(adev,
3803                                    pi->dpm_table_start +
3804                                    offsetof(SMU7_Discrete_DpmTable, SystemFlags),
3805                                    (u8 *)&table->SystemFlags,
3806                                    sizeof(SMU7_Discrete_DpmTable) - 3 * sizeof(SMU7_PIDController),
3807                                    pi->sram_end);
3808         if (ret)
3809                 return ret;
3810
3811         return 0;
3812 }
3813
3814 static void ci_trim_single_dpm_states(struct amdgpu_device *adev,
3815                                       struct ci_single_dpm_table *dpm_table,
3816                                       u32 low_limit, u32 high_limit)
3817 {
3818         u32 i;
3819
3820         for (i = 0; i < dpm_table->count; i++) {
3821                 if ((dpm_table->dpm_levels[i].value < low_limit) ||
3822                     (dpm_table->dpm_levels[i].value > high_limit))
3823                         dpm_table->dpm_levels[i].enabled = false;
3824                 else
3825                         dpm_table->dpm_levels[i].enabled = true;
3826         }
3827 }
3828
3829 static void ci_trim_pcie_dpm_states(struct amdgpu_device *adev,
3830                                     u32 speed_low, u32 lanes_low,
3831                                     u32 speed_high, u32 lanes_high)
3832 {
3833         struct ci_power_info *pi = ci_get_pi(adev);
3834         struct ci_single_dpm_table *pcie_table = &pi->dpm_table.pcie_speed_table;
3835         u32 i, j;
3836
3837         for (i = 0; i < pcie_table->count; i++) {
3838                 if ((pcie_table->dpm_levels[i].value < speed_low) ||
3839                     (pcie_table->dpm_levels[i].param1 < lanes_low) ||
3840                     (pcie_table->dpm_levels[i].value > speed_high) ||
3841                     (pcie_table->dpm_levels[i].param1 > lanes_high))
3842                         pcie_table->dpm_levels[i].enabled = false;
3843                 else
3844                         pcie_table->dpm_levels[i].enabled = true;
3845         }
3846
3847         for (i = 0; i < pcie_table->count; i++) {
3848                 if (pcie_table->dpm_levels[i].enabled) {
3849                         for (j = i + 1; j < pcie_table->count; j++) {
3850                                 if (pcie_table->dpm_levels[j].enabled) {
3851                                         if ((pcie_table->dpm_levels[i].value == pcie_table->dpm_levels[j].value) &&
3852                                             (pcie_table->dpm_levels[i].param1 == pcie_table->dpm_levels[j].param1))
3853                                                 pcie_table->dpm_levels[j].enabled = false;
3854                                 }
3855                         }
3856                 }
3857         }
3858 }
3859
3860 static int ci_trim_dpm_states(struct amdgpu_device *adev,
3861                               struct amdgpu_ps *amdgpu_state)
3862 {
3863         struct ci_ps *state = ci_get_ps(amdgpu_state);
3864         struct ci_power_info *pi = ci_get_pi(adev);
3865         u32 high_limit_count;
3866
3867         if (state->performance_level_count < 1)
3868                 return -EINVAL;
3869
3870         if (state->performance_level_count == 1)
3871                 high_limit_count = 0;
3872         else
3873                 high_limit_count = 1;
3874
3875         ci_trim_single_dpm_states(adev,
3876                                   &pi->dpm_table.sclk_table,
3877                                   state->performance_levels[0].sclk,
3878                                   state->performance_levels[high_limit_count].sclk);
3879
3880         ci_trim_single_dpm_states(adev,
3881                                   &pi->dpm_table.mclk_table,
3882                                   state->performance_levels[0].mclk,
3883                                   state->performance_levels[high_limit_count].mclk);
3884
3885         ci_trim_pcie_dpm_states(adev,
3886                                 state->performance_levels[0].pcie_gen,
3887                                 state->performance_levels[0].pcie_lane,
3888                                 state->performance_levels[high_limit_count].pcie_gen,
3889                                 state->performance_levels[high_limit_count].pcie_lane);
3890
3891         return 0;
3892 }
3893
3894 static int ci_apply_disp_minimum_voltage_request(struct amdgpu_device *adev)
3895 {
3896         struct amdgpu_clock_voltage_dependency_table *disp_voltage_table =
3897                 &adev->pm.dpm.dyn_state.vddc_dependency_on_dispclk;
3898         struct amdgpu_clock_voltage_dependency_table *vddc_table =
3899                 &adev->pm.dpm.dyn_state.vddc_dependency_on_sclk;
3900         u32 requested_voltage = 0;
3901         u32 i;
3902
3903         if (disp_voltage_table == NULL)
3904                 return -EINVAL;
3905         if (!disp_voltage_table->count)
3906                 return -EINVAL;
3907
3908         for (i = 0; i < disp_voltage_table->count; i++) {
3909                 if (adev->clock.current_dispclk == disp_voltage_table->entries[i].clk)
3910                         requested_voltage = disp_voltage_table->entries[i].v;
3911         }
3912
3913         for (i = 0; i < vddc_table->count; i++) {
3914                 if (requested_voltage <= vddc_table->entries[i].v) {
3915                         requested_voltage = vddc_table->entries[i].v;
3916                         return (amdgpu_ci_send_msg_to_smc_with_parameter(adev,
3917                                                                   PPSMC_MSG_VddC_Request,
3918                                                                   requested_voltage * VOLTAGE_SCALE) == PPSMC_Result_OK) ?
3919                                 0 : -EINVAL;
3920                 }
3921         }
3922
3923         return -EINVAL;
3924 }
3925
3926 static int ci_upload_dpm_level_enable_mask(struct amdgpu_device *adev)
3927 {
3928         struct ci_power_info *pi = ci_get_pi(adev);
3929         PPSMC_Result result;
3930
3931         ci_apply_disp_minimum_voltage_request(adev);
3932
3933         if (!pi->sclk_dpm_key_disabled) {
3934                 if (pi->dpm_level_enable_mask.sclk_dpm_enable_mask) {
3935                         result = amdgpu_ci_send_msg_to_smc_with_parameter(adev,
3936                                                                    PPSMC_MSG_SCLKDPM_SetEnabledMask,
3937                                                                    pi->dpm_level_enable_mask.sclk_dpm_enable_mask);
3938                         if (result != PPSMC_Result_OK)
3939                                 return -EINVAL;
3940                 }
3941         }
3942
3943         if (!pi->mclk_dpm_key_disabled) {
3944                 if (pi->dpm_level_enable_mask.mclk_dpm_enable_mask) {
3945                         result = amdgpu_ci_send_msg_to_smc_with_parameter(adev,
3946                                                                    PPSMC_MSG_MCLKDPM_SetEnabledMask,
3947                                                                    pi->dpm_level_enable_mask.mclk_dpm_enable_mask);
3948                         if (result != PPSMC_Result_OK)
3949                                 return -EINVAL;
3950                 }
3951         }
3952
3953 #if 0
3954         if (!pi->pcie_dpm_key_disabled) {
3955                 if (pi->dpm_level_enable_mask.pcie_dpm_enable_mask) {
3956                         result = amdgpu_ci_send_msg_to_smc_with_parameter(adev,
3957                                                                    PPSMC_MSG_PCIeDPM_SetEnabledMask,
3958                                                                    pi->dpm_level_enable_mask.pcie_dpm_enable_mask);
3959                         if (result != PPSMC_Result_OK)
3960                                 return -EINVAL;
3961                 }
3962         }
3963 #endif
3964
3965         return 0;
3966 }
3967
3968 static void ci_find_dpm_states_clocks_in_dpm_table(struct amdgpu_device *adev,
3969                                                    struct amdgpu_ps *amdgpu_state)
3970 {
3971         struct ci_power_info *pi = ci_get_pi(adev);
3972         struct ci_ps *state = ci_get_ps(amdgpu_state);
3973         struct ci_single_dpm_table *sclk_table = &pi->dpm_table.sclk_table;
3974         u32 sclk = state->performance_levels[state->performance_level_count-1].sclk;
3975         struct ci_single_dpm_table *mclk_table = &pi->dpm_table.mclk_table;
3976         u32 mclk = state->performance_levels[state->performance_level_count-1].mclk;
3977         u32 i;
3978
3979         pi->need_update_smu7_dpm_table = 0;
3980
3981         for (i = 0; i < sclk_table->count; i++) {
3982                 if (sclk == sclk_table->dpm_levels[i].value)
3983                         break;
3984         }
3985
3986         if (i >= sclk_table->count) {
3987                 pi->need_update_smu7_dpm_table |= DPMTABLE_OD_UPDATE_SCLK;
3988         } else {
3989                 /* XXX check display min clock requirements */
3990                 if (CISLAND_MINIMUM_ENGINE_CLOCK != CISLAND_MINIMUM_ENGINE_CLOCK)
3991                         pi->need_update_smu7_dpm_table |= DPMTABLE_UPDATE_SCLK;
3992         }
3993
3994         for (i = 0; i < mclk_table->count; i++) {
3995                 if (mclk == mclk_table->dpm_levels[i].value)
3996                         break;
3997         }
3998
3999         if (i >= mclk_table->count)
4000                 pi->need_update_smu7_dpm_table |= DPMTABLE_OD_UPDATE_MCLK;
4001
4002         if (adev->pm.dpm.current_active_crtc_count !=
4003             adev->pm.dpm.new_active_crtc_count)
4004                 pi->need_update_smu7_dpm_table |= DPMTABLE_UPDATE_MCLK;
4005 }
4006
4007 static int ci_populate_and_upload_sclk_mclk_dpm_levels(struct amdgpu_device *adev,
4008                                                        struct amdgpu_ps *amdgpu_state)
4009 {
4010         struct ci_power_info *pi = ci_get_pi(adev);
4011         struct ci_ps *state = ci_get_ps(amdgpu_state);
4012         u32 sclk = state->performance_levels[state->performance_level_count-1].sclk;
4013         u32 mclk = state->performance_levels[state->performance_level_count-1].mclk;
4014         struct ci_dpm_table *dpm_table = &pi->dpm_table;
4015         int ret;
4016
4017         if (!pi->need_update_smu7_dpm_table)
4018                 return 0;
4019
4020         if (pi->need_update_smu7_dpm_table & DPMTABLE_OD_UPDATE_SCLK)
4021                 dpm_table->sclk_table.dpm_levels[dpm_table->sclk_table.count-1].value = sclk;
4022
4023         if (pi->need_update_smu7_dpm_table & DPMTABLE_OD_UPDATE_MCLK)
4024                 dpm_table->mclk_table.dpm_levels[dpm_table->mclk_table.count-1].value = mclk;
4025
4026         if (pi->need_update_smu7_dpm_table & (DPMTABLE_OD_UPDATE_SCLK | DPMTABLE_UPDATE_SCLK)) {
4027                 ret = ci_populate_all_graphic_levels(adev);
4028                 if (ret)
4029                         return ret;
4030         }
4031
4032         if (pi->need_update_smu7_dpm_table & (DPMTABLE_OD_UPDATE_MCLK | DPMTABLE_UPDATE_MCLK)) {
4033                 ret = ci_populate_all_memory_levels(adev);
4034                 if (ret)
4035                         return ret;
4036         }
4037
4038         return 0;
4039 }
4040
4041 static int ci_enable_uvd_dpm(struct amdgpu_device *adev, bool enable)
4042 {
4043         struct ci_power_info *pi = ci_get_pi(adev);
4044         const struct amdgpu_clock_and_voltage_limits *max_limits;
4045         int i;
4046
4047         if (adev->pm.dpm.ac_power)
4048                 max_limits = &adev->pm.dpm.dyn_state.max_clock_voltage_on_ac;
4049         else
4050                 max_limits = &adev->pm.dpm.dyn_state.max_clock_voltage_on_dc;
4051
4052         if (enable) {
4053                 pi->dpm_level_enable_mask.uvd_dpm_enable_mask = 0;
4054
4055                 for (i = adev->pm.dpm.dyn_state.uvd_clock_voltage_dependency_table.count - 1; i >= 0; i--) {
4056                         if (adev->pm.dpm.dyn_state.uvd_clock_voltage_dependency_table.entries[i].v <= max_limits->vddc) {
4057                                 pi->dpm_level_enable_mask.uvd_dpm_enable_mask |= 1 << i;
4058
4059                                 if (!pi->caps_uvd_dpm)
4060                                         break;
4061                         }
4062                 }
4063
4064                 amdgpu_ci_send_msg_to_smc_with_parameter(adev,
4065                                                   PPSMC_MSG_UVDDPM_SetEnabledMask,
4066                                                   pi->dpm_level_enable_mask.uvd_dpm_enable_mask);
4067
4068                 if (pi->last_mclk_dpm_enable_mask & 0x1) {
4069                         pi->uvd_enabled = true;
4070                         pi->dpm_level_enable_mask.mclk_dpm_enable_mask &= 0xFFFFFFFE;
4071                         amdgpu_ci_send_msg_to_smc_with_parameter(adev,
4072                                                           PPSMC_MSG_MCLKDPM_SetEnabledMask,
4073                                                           pi->dpm_level_enable_mask.mclk_dpm_enable_mask);
4074                 }
4075         } else {
4076                 if (pi->last_mclk_dpm_enable_mask & 0x1) {
4077                         pi->uvd_enabled = false;
4078                         pi->dpm_level_enable_mask.mclk_dpm_enable_mask |= 1;
4079                         amdgpu_ci_send_msg_to_smc_with_parameter(adev,
4080                                                           PPSMC_MSG_MCLKDPM_SetEnabledMask,
4081                                                           pi->dpm_level_enable_mask.mclk_dpm_enable_mask);
4082                 }
4083         }
4084
4085         return (amdgpu_ci_send_msg_to_smc(adev, enable ?
4086                                    PPSMC_MSG_UVDDPM_Enable : PPSMC_MSG_UVDDPM_Disable) == PPSMC_Result_OK) ?
4087                 0 : -EINVAL;
4088 }
4089
4090 static int ci_enable_vce_dpm(struct amdgpu_device *adev, bool enable)
4091 {
4092         struct ci_power_info *pi = ci_get_pi(adev);
4093         const struct amdgpu_clock_and_voltage_limits *max_limits;
4094         int i;
4095
4096         if (adev->pm.dpm.ac_power)
4097                 max_limits = &adev->pm.dpm.dyn_state.max_clock_voltage_on_ac;
4098         else
4099                 max_limits = &adev->pm.dpm.dyn_state.max_clock_voltage_on_dc;
4100
4101         if (enable) {
4102                 pi->dpm_level_enable_mask.vce_dpm_enable_mask = 0;
4103                 for (i = adev->pm.dpm.dyn_state.vce_clock_voltage_dependency_table.count - 1; i >= 0; i--) {
4104                         if (adev->pm.dpm.dyn_state.vce_clock_voltage_dependency_table.entries[i].v <= max_limits->vddc) {
4105                                 pi->dpm_level_enable_mask.vce_dpm_enable_mask |= 1 << i;
4106
4107                                 if (!pi->caps_vce_dpm)
4108                                         break;
4109                         }
4110                 }
4111
4112                 amdgpu_ci_send_msg_to_smc_with_parameter(adev,
4113                                                   PPSMC_MSG_VCEDPM_SetEnabledMask,
4114                                                   pi->dpm_level_enable_mask.vce_dpm_enable_mask);
4115         }
4116
4117         return (amdgpu_ci_send_msg_to_smc(adev, enable ?
4118                                    PPSMC_MSG_VCEDPM_Enable : PPSMC_MSG_VCEDPM_Disable) == PPSMC_Result_OK) ?
4119                 0 : -EINVAL;
4120 }
4121
4122 #if 0
4123 static int ci_enable_samu_dpm(struct amdgpu_device *adev, bool enable)
4124 {
4125         struct ci_power_info *pi = ci_get_pi(adev);
4126         const struct amdgpu_clock_and_voltage_limits *max_limits;
4127         int i;
4128
4129         if (adev->pm.dpm.ac_power)
4130                 max_limits = &adev->pm.dpm.dyn_state.max_clock_voltage_on_ac;
4131         else
4132                 max_limits = &adev->pm.dpm.dyn_state.max_clock_voltage_on_dc;
4133
4134         if (enable) {
4135                 pi->dpm_level_enable_mask.samu_dpm_enable_mask = 0;
4136                 for (i = adev->pm.dpm.dyn_state.samu_clock_voltage_dependency_table.count - 1; i >= 0; i--) {
4137                         if (adev->pm.dpm.dyn_state.samu_clock_voltage_dependency_table.entries[i].v <= max_limits->vddc) {
4138                                 pi->dpm_level_enable_mask.samu_dpm_enable_mask |= 1 << i;
4139
4140                                 if (!pi->caps_samu_dpm)
4141                                         break;
4142                         }
4143                 }
4144
4145                 amdgpu_ci_send_msg_to_smc_with_parameter(adev,
4146                                                   PPSMC_MSG_SAMUDPM_SetEnabledMask,
4147                                                   pi->dpm_level_enable_mask.samu_dpm_enable_mask);
4148         }
4149         return (amdgpu_ci_send_msg_to_smc(adev, enable ?
4150                                    PPSMC_MSG_SAMUDPM_Enable : PPSMC_MSG_SAMUDPM_Disable) == PPSMC_Result_OK) ?
4151                 0 : -EINVAL;
4152 }
4153
4154 static int ci_enable_acp_dpm(struct amdgpu_device *adev, bool enable)
4155 {
4156         struct ci_power_info *pi = ci_get_pi(adev);
4157         const struct amdgpu_clock_and_voltage_limits *max_limits;
4158         int i;
4159
4160         if (adev->pm.dpm.ac_power)
4161                 max_limits = &adev->pm.dpm.dyn_state.max_clock_voltage_on_ac;
4162         else
4163                 max_limits = &adev->pm.dpm.dyn_state.max_clock_voltage_on_dc;
4164
4165         if (enable) {
4166                 pi->dpm_level_enable_mask.acp_dpm_enable_mask = 0;
4167                 for (i = adev->pm.dpm.dyn_state.acp_clock_voltage_dependency_table.count - 1; i >= 0; i--) {
4168                         if (adev->pm.dpm.dyn_state.acp_clock_voltage_dependency_table.entries[i].v <= max_limits->vddc) {
4169                                 pi->dpm_level_enable_mask.acp_dpm_enable_mask |= 1 << i;
4170
4171                                 if (!pi->caps_acp_dpm)
4172                                         break;
4173                         }
4174                 }
4175
4176                 amdgpu_ci_send_msg_to_smc_with_parameter(adev,
4177                                                   PPSMC_MSG_ACPDPM_SetEnabledMask,
4178                                                   pi->dpm_level_enable_mask.acp_dpm_enable_mask);
4179         }
4180
4181         return (amdgpu_ci_send_msg_to_smc(adev, enable ?
4182                                    PPSMC_MSG_ACPDPM_Enable : PPSMC_MSG_ACPDPM_Disable) == PPSMC_Result_OK) ?
4183                 0 : -EINVAL;
4184 }
4185 #endif
4186
4187 static int ci_update_uvd_dpm(struct amdgpu_device *adev, bool gate)
4188 {
4189         struct ci_power_info *pi = ci_get_pi(adev);
4190         u32 tmp;
4191
4192         if (!gate) {
4193                 if (pi->caps_uvd_dpm ||
4194                     (adev->pm.dpm.dyn_state.uvd_clock_voltage_dependency_table.count <= 0))
4195                         pi->smc_state_table.UvdBootLevel = 0;
4196                 else
4197                         pi->smc_state_table.UvdBootLevel =
4198                                 adev->pm.dpm.dyn_state.uvd_clock_voltage_dependency_table.count - 1;
4199
4200                 tmp = RREG32_SMC(ixDPM_TABLE_475);
4201                 tmp &= ~DPM_TABLE_475__UvdBootLevel_MASK;
4202                 tmp |= (pi->smc_state_table.UvdBootLevel << DPM_TABLE_475__UvdBootLevel__SHIFT);
4203                 WREG32_SMC(ixDPM_TABLE_475, tmp);
4204         }
4205
4206         return ci_enable_uvd_dpm(adev, !gate);
4207 }
4208
4209 static u8 ci_get_vce_boot_level(struct amdgpu_device *adev)
4210 {
4211         u8 i;
4212         u32 min_evclk = 30000; /* ??? */
4213         struct amdgpu_vce_clock_voltage_dependency_table *table =
4214                 &adev->pm.dpm.dyn_state.vce_clock_voltage_dependency_table;
4215
4216         for (i = 0; i < table->count; i++) {
4217                 if (table->entries[i].evclk >= min_evclk)
4218                         return i;
4219         }
4220
4221         return table->count - 1;
4222 }
4223
4224 static int ci_update_vce_dpm(struct amdgpu_device *adev,
4225                              struct amdgpu_ps *amdgpu_new_state,
4226                              struct amdgpu_ps *amdgpu_current_state)
4227 {
4228         struct ci_power_info *pi = ci_get_pi(adev);
4229         int ret = 0;
4230         u32 tmp;
4231
4232         if (amdgpu_current_state->evclk != amdgpu_new_state->evclk) {
4233                 if (amdgpu_new_state->evclk) {
4234                         /* turn the clocks on when encoding */
4235                         ret = amdgpu_set_clockgating_state(adev, AMD_IP_BLOCK_TYPE_VCE,
4236                                                             AMD_CG_STATE_UNGATE);
4237                         if (ret)
4238                                 return ret;
4239
4240                         pi->smc_state_table.VceBootLevel = ci_get_vce_boot_level(adev);
4241                         tmp = RREG32_SMC(ixDPM_TABLE_475);
4242                         tmp &= ~DPM_TABLE_475__VceBootLevel_MASK;
4243                         tmp |= (pi->smc_state_table.VceBootLevel << DPM_TABLE_475__VceBootLevel__SHIFT);
4244                         WREG32_SMC(ixDPM_TABLE_475, tmp);
4245
4246                         ret = ci_enable_vce_dpm(adev, true);
4247                 } else {
4248                         /* turn the clocks off when not encoding */
4249                         ret = amdgpu_set_clockgating_state(adev, AMD_IP_BLOCK_TYPE_VCE,
4250                                                             AMD_CG_STATE_GATE);
4251                         if (ret)
4252                                 return ret;
4253
4254                         ret = ci_enable_vce_dpm(adev, false);
4255                 }
4256         }
4257         return ret;
4258 }
4259
4260 #if 0
4261 static int ci_update_samu_dpm(struct amdgpu_device *adev, bool gate)
4262 {
4263         return ci_enable_samu_dpm(adev, gate);
4264 }
4265
4266 static int ci_update_acp_dpm(struct amdgpu_device *adev, bool gate)
4267 {
4268         struct ci_power_info *pi = ci_get_pi(adev);
4269         u32 tmp;
4270
4271         if (!gate) {
4272                 pi->smc_state_table.AcpBootLevel = 0;
4273
4274                 tmp = RREG32_SMC(ixDPM_TABLE_475);
4275                 tmp &= ~AcpBootLevel_MASK;
4276                 tmp |= AcpBootLevel(pi->smc_state_table.AcpBootLevel);
4277                 WREG32_SMC(ixDPM_TABLE_475, tmp);
4278         }
4279
4280         return ci_enable_acp_dpm(adev, !gate);
4281 }
4282 #endif
4283
4284 static int ci_generate_dpm_level_enable_mask(struct amdgpu_device *adev,
4285                                              struct amdgpu_ps *amdgpu_state)
4286 {
4287         struct ci_power_info *pi = ci_get_pi(adev);
4288         int ret;
4289
4290         ret = ci_trim_dpm_states(adev, amdgpu_state);
4291         if (ret)
4292                 return ret;
4293
4294         pi->dpm_level_enable_mask.sclk_dpm_enable_mask =
4295                 ci_get_dpm_level_enable_mask_value(&pi->dpm_table.sclk_table);
4296         pi->dpm_level_enable_mask.mclk_dpm_enable_mask =
4297                 ci_get_dpm_level_enable_mask_value(&pi->dpm_table.mclk_table);
4298         pi->last_mclk_dpm_enable_mask =
4299                 pi->dpm_level_enable_mask.mclk_dpm_enable_mask;
4300         if (pi->uvd_enabled) {
4301                 if (pi->dpm_level_enable_mask.mclk_dpm_enable_mask & 1)
4302                         pi->dpm_level_enable_mask.mclk_dpm_enable_mask &= 0xFFFFFFFE;
4303         }
4304         pi->dpm_level_enable_mask.pcie_dpm_enable_mask =
4305                 ci_get_dpm_level_enable_mask_value(&pi->dpm_table.pcie_speed_table);
4306
4307         return 0;
4308 }
4309
4310 static u32 ci_get_lowest_enabled_level(struct amdgpu_device *adev,
4311                                        u32 level_mask)
4312 {
4313         u32 level = 0;
4314
4315         while ((level_mask & (1 << level)) == 0)
4316                 level++;
4317
4318         return level;
4319 }
4320
4321
4322 static int ci_dpm_force_performance_level(struct amdgpu_device *adev,
4323                                           enum amdgpu_dpm_forced_level level)
4324 {
4325         struct ci_power_info *pi = ci_get_pi(adev);
4326         u32 tmp, levels, i;
4327         int ret;
4328
4329         if (level == AMDGPU_DPM_FORCED_LEVEL_HIGH) {
4330                 if ((!pi->pcie_dpm_key_disabled) &&
4331                     pi->dpm_level_enable_mask.pcie_dpm_enable_mask) {
4332                         levels = 0;
4333                         tmp = pi->dpm_level_enable_mask.pcie_dpm_enable_mask;
4334                         while (tmp >>= 1)
4335                                 levels++;
4336                         if (levels) {
4337                                 ret = ci_dpm_force_state_pcie(adev, level);
4338                                 if (ret)
4339                                         return ret;
4340                                 for (i = 0; i < adev->usec_timeout; i++) {
4341                                         tmp = (RREG32_SMC(ixTARGET_AND_CURRENT_PROFILE_INDEX_1) &
4342                                         TARGET_AND_CURRENT_PROFILE_INDEX_1__CURR_PCIE_INDEX_MASK) >>
4343                                         TARGET_AND_CURRENT_PROFILE_INDEX_1__CURR_PCIE_INDEX__SHIFT;
4344                                         if (tmp == levels)
4345                                                 break;
4346                                         udelay(1);
4347                                 }
4348                         }
4349                 }
4350                 if ((!pi->sclk_dpm_key_disabled) &&
4351                     pi->dpm_level_enable_mask.sclk_dpm_enable_mask) {
4352                         levels = 0;
4353                         tmp = pi->dpm_level_enable_mask.sclk_dpm_enable_mask;
4354                         while (tmp >>= 1)
4355                                 levels++;
4356                         if (levels) {
4357                                 ret = ci_dpm_force_state_sclk(adev, levels);
4358                                 if (ret)
4359                                         return ret;
4360                                 for (i = 0; i < adev->usec_timeout; i++) {
4361                                         tmp = (RREG32_SMC(ixTARGET_AND_CURRENT_PROFILE_INDEX) &
4362                                         TARGET_AND_CURRENT_PROFILE_INDEX__CURR_SCLK_INDEX_MASK) >>
4363                                         TARGET_AND_CURRENT_PROFILE_INDEX__CURR_SCLK_INDEX__SHIFT;
4364                                         if (tmp == levels)
4365                                                 break;
4366                                         udelay(1);
4367                                 }
4368                         }
4369                 }
4370                 if ((!pi->mclk_dpm_key_disabled) &&
4371                     pi->dpm_level_enable_mask.mclk_dpm_enable_mask) {
4372                         levels = 0;
4373                         tmp = pi->dpm_level_enable_mask.mclk_dpm_enable_mask;
4374                         while (tmp >>= 1)
4375                                 levels++;
4376                         if (levels) {
4377                                 ret = ci_dpm_force_state_mclk(adev, levels);
4378                                 if (ret)
4379                                         return ret;
4380                                 for (i = 0; i < adev->usec_timeout; i++) {
4381                                         tmp = (RREG32_SMC(ixTARGET_AND_CURRENT_PROFILE_INDEX) &
4382                                         TARGET_AND_CURRENT_PROFILE_INDEX__CURR_MCLK_INDEX_MASK) >>
4383                                         TARGET_AND_CURRENT_PROFILE_INDEX__CURR_MCLK_INDEX__SHIFT;
4384                                         if (tmp == levels)
4385                                                 break;
4386                                         udelay(1);
4387                                 }
4388                         }
4389                 }
4390                 if ((!pi->pcie_dpm_key_disabled) &&
4391                     pi->dpm_level_enable_mask.pcie_dpm_enable_mask) {
4392                         levels = 0;
4393                         tmp = pi->dpm_level_enable_mask.pcie_dpm_enable_mask;
4394                         while (tmp >>= 1)
4395                                 levels++;
4396                         if (levels) {
4397                                 ret = ci_dpm_force_state_pcie(adev, level);
4398                                 if (ret)
4399                                         return ret;
4400                                 for (i = 0; i < adev->usec_timeout; i++) {
4401                                         tmp = (RREG32_SMC(ixTARGET_AND_CURRENT_PROFILE_INDEX_1) &
4402                                         TARGET_AND_CURRENT_PROFILE_INDEX_1__CURR_PCIE_INDEX_MASK) >>
4403                                         TARGET_AND_CURRENT_PROFILE_INDEX_1__CURR_PCIE_INDEX__SHIFT;
4404                                         if (tmp == levels)
4405                                                 break;
4406                                         udelay(1);
4407                                 }
4408                         }
4409                 }
4410         } else if (level == AMDGPU_DPM_FORCED_LEVEL_LOW) {
4411                 if ((!pi->sclk_dpm_key_disabled) &&
4412                     pi->dpm_level_enable_mask.sclk_dpm_enable_mask) {
4413                         levels = ci_get_lowest_enabled_level(adev,
4414                                                              pi->dpm_level_enable_mask.sclk_dpm_enable_mask);
4415                         ret = ci_dpm_force_state_sclk(adev, levels);
4416                         if (ret)
4417                                 return ret;
4418                         for (i = 0; i < adev->usec_timeout; i++) {
4419                                 tmp = (RREG32_SMC(ixTARGET_AND_CURRENT_PROFILE_INDEX) &
4420                                 TARGET_AND_CURRENT_PROFILE_INDEX__CURR_SCLK_INDEX_MASK) >>
4421                                 TARGET_AND_CURRENT_PROFILE_INDEX__CURR_SCLK_INDEX__SHIFT;
4422                                 if (tmp == levels)
4423                                         break;
4424                                 udelay(1);
4425                         }
4426                 }
4427                 if ((!pi->mclk_dpm_key_disabled) &&
4428                     pi->dpm_level_enable_mask.mclk_dpm_enable_mask) {
4429                         levels = ci_get_lowest_enabled_level(adev,
4430                                                              pi->dpm_level_enable_mask.mclk_dpm_enable_mask);
4431                         ret = ci_dpm_force_state_mclk(adev, levels);
4432                         if (ret)
4433                                 return ret;
4434                         for (i = 0; i < adev->usec_timeout; i++) {
4435                                 tmp = (RREG32_SMC(ixTARGET_AND_CURRENT_PROFILE_INDEX) &
4436                                 TARGET_AND_CURRENT_PROFILE_INDEX__CURR_MCLK_INDEX_MASK) >>
4437                                 TARGET_AND_CURRENT_PROFILE_INDEX__CURR_MCLK_INDEX__SHIFT;
4438                                 if (tmp == levels)
4439                                         break;
4440                                 udelay(1);
4441                         }
4442                 }
4443                 if ((!pi->pcie_dpm_key_disabled) &&
4444                     pi->dpm_level_enable_mask.pcie_dpm_enable_mask) {
4445                         levels = ci_get_lowest_enabled_level(adev,
4446                                                              pi->dpm_level_enable_mask.pcie_dpm_enable_mask);
4447                         ret = ci_dpm_force_state_pcie(adev, levels);
4448                         if (ret)
4449                                 return ret;
4450                         for (i = 0; i < adev->usec_timeout; i++) {
4451                                 tmp = (RREG32_SMC(ixTARGET_AND_CURRENT_PROFILE_INDEX_1) &
4452                                 TARGET_AND_CURRENT_PROFILE_INDEX_1__CURR_PCIE_INDEX_MASK) >>
4453                                 TARGET_AND_CURRENT_PROFILE_INDEX_1__CURR_PCIE_INDEX__SHIFT;
4454                                 if (tmp == levels)
4455                                         break;
4456                                 udelay(1);
4457                         }
4458                 }
4459         } else if (level == AMDGPU_DPM_FORCED_LEVEL_AUTO) {
4460                 if (!pi->pcie_dpm_key_disabled) {
4461                         PPSMC_Result smc_result;
4462
4463                         smc_result = amdgpu_ci_send_msg_to_smc(adev,
4464                                                                PPSMC_MSG_PCIeDPM_UnForceLevel);
4465                         if (smc_result != PPSMC_Result_OK)
4466                                 return -EINVAL;
4467                 }
4468                 ret = ci_upload_dpm_level_enable_mask(adev);
4469                 if (ret)
4470                         return ret;
4471         }
4472
4473         adev->pm.dpm.forced_level = level;
4474
4475         return 0;
4476 }
4477
4478 static int ci_set_mc_special_registers(struct amdgpu_device *adev,
4479                                        struct ci_mc_reg_table *table)
4480 {
4481         u8 i, j, k;
4482         u32 temp_reg;
4483
4484         for (i = 0, j = table->last; i < table->last; i++) {
4485                 if (j >= SMU7_DISCRETE_MC_REGISTER_ARRAY_SIZE)
4486                         return -EINVAL;
4487                 switch(table->mc_reg_address[i].s1) {
4488                 case mmMC_SEQ_MISC1:
4489                         temp_reg = RREG32(mmMC_PMG_CMD_EMRS);
4490                         table->mc_reg_address[j].s1 = mmMC_PMG_CMD_EMRS;
4491                         table->mc_reg_address[j].s0 = mmMC_SEQ_PMG_CMD_EMRS_LP;
4492                         for (k = 0; k < table->num_entries; k++) {
4493                                 table->mc_reg_table_entry[k].mc_data[j] =
4494                                         ((temp_reg & 0xffff0000)) | ((table->mc_reg_table_entry[k].mc_data[i] & 0xffff0000) >> 16);
4495                         }
4496                         j++;
4497                         if (j >= SMU7_DISCRETE_MC_REGISTER_ARRAY_SIZE)
4498                                 return -EINVAL;
4499
4500                         temp_reg = RREG32(mmMC_PMG_CMD_MRS);
4501                         table->mc_reg_address[j].s1 = mmMC_PMG_CMD_MRS;
4502                         table->mc_reg_address[j].s0 = mmMC_SEQ_PMG_CMD_MRS_LP;
4503                         for (k = 0; k < table->num_entries; k++) {
4504                                 table->mc_reg_table_entry[k].mc_data[j] =
4505                                         (temp_reg & 0xffff0000) | (table->mc_reg_table_entry[k].mc_data[i] & 0x0000ffff);
4506                                 if (adev->mc.vram_type != AMDGPU_VRAM_TYPE_GDDR5)
4507                                         table->mc_reg_table_entry[k].mc_data[j] |= 0x100;
4508                         }
4509                         j++;
4510                         if (j > SMU7_DISCRETE_MC_REGISTER_ARRAY_SIZE)
4511                                 return -EINVAL;
4512
4513                         if (adev->mc.vram_type != AMDGPU_VRAM_TYPE_GDDR5) {
4514                                 table->mc_reg_address[j].s1 = mmMC_PMG_AUTO_CMD;
4515                                 table->mc_reg_address[j].s0 = mmMC_PMG_AUTO_CMD;
4516                                 for (k = 0; k < table->num_entries; k++) {
4517                                         table->mc_reg_table_entry[k].mc_data[j] =
4518                                                 (table->mc_reg_table_entry[k].mc_data[i] & 0xffff0000) >> 16;
4519                                 }
4520                                 j++;
4521                                 if (j > SMU7_DISCRETE_MC_REGISTER_ARRAY_SIZE)
4522                                         return -EINVAL;
4523                         }
4524                         break;
4525                 case mmMC_SEQ_RESERVE_M:
4526                         temp_reg = RREG32(mmMC_PMG_CMD_MRS1);
4527                         table->mc_reg_address[j].s1 = mmMC_PMG_CMD_MRS1;
4528                         table->mc_reg_address[j].s0 = mmMC_SEQ_PMG_CMD_MRS1_LP;
4529                         for (k = 0; k < table->num_entries; k++) {
4530                                 table->mc_reg_table_entry[k].mc_data[j] =
4531                                         (temp_reg & 0xffff0000) | (table->mc_reg_table_entry[k].mc_data[i] & 0x0000ffff);
4532                         }
4533                         j++;
4534                         if (j > SMU7_DISCRETE_MC_REGISTER_ARRAY_SIZE)
4535                                 return -EINVAL;
4536                         break;
4537                 default:
4538                         break;
4539                 }
4540
4541         }
4542
4543         table->last = j;
4544
4545         return 0;
4546 }
4547
4548 static bool ci_check_s0_mc_reg_index(u16 in_reg, u16 *out_reg)
4549 {
4550         bool result = true;
4551
4552         switch(in_reg) {
4553         case mmMC_SEQ_RAS_TIMING:
4554                 *out_reg = mmMC_SEQ_RAS_TIMING_LP;
4555                 break;
4556         case mmMC_SEQ_DLL_STBY:
4557                 *out_reg = mmMC_SEQ_DLL_STBY_LP;
4558                 break;
4559         case mmMC_SEQ_G5PDX_CMD0:
4560                 *out_reg = mmMC_SEQ_G5PDX_CMD0_LP;
4561                 break;
4562         case mmMC_SEQ_G5PDX_CMD1:
4563                 *out_reg = mmMC_SEQ_G5PDX_CMD1_LP;
4564                 break;
4565         case mmMC_SEQ_G5PDX_CTRL:
4566                 *out_reg = mmMC_SEQ_G5PDX_CTRL_LP;
4567                 break;
4568         case mmMC_SEQ_CAS_TIMING:
4569                 *out_reg = mmMC_SEQ_CAS_TIMING_LP;
4570             break;
4571         case mmMC_SEQ_MISC_TIMING:
4572                 *out_reg = mmMC_SEQ_MISC_TIMING_LP;
4573                 break;
4574         case mmMC_SEQ_MISC_TIMING2:
4575                 *out_reg = mmMC_SEQ_MISC_TIMING2_LP;
4576                 break;
4577         case mmMC_SEQ_PMG_DVS_CMD:
4578                 *out_reg = mmMC_SEQ_PMG_DVS_CMD_LP;
4579                 break;
4580         case mmMC_SEQ_PMG_DVS_CTL:
4581                 *out_reg = mmMC_SEQ_PMG_DVS_CTL_LP;
4582                 break;
4583         case mmMC_SEQ_RD_CTL_D0:
4584                 *out_reg = mmMC_SEQ_RD_CTL_D0_LP;
4585                 break;
4586         case mmMC_SEQ_RD_CTL_D1:
4587                 *out_reg = mmMC_SEQ_RD_CTL_D1_LP;
4588                 break;
4589         case mmMC_SEQ_WR_CTL_D0:
4590                 *out_reg = mmMC_SEQ_WR_CTL_D0_LP;
4591                 break;
4592         case mmMC_SEQ_WR_CTL_D1:
4593                 *out_reg = mmMC_SEQ_WR_CTL_D1_LP;
4594                 break;
4595         case mmMC_PMG_CMD_EMRS:
4596                 *out_reg = mmMC_SEQ_PMG_CMD_EMRS_LP;
4597                 break;
4598         case mmMC_PMG_CMD_MRS:
4599                 *out_reg = mmMC_SEQ_PMG_CMD_MRS_LP;
4600                 break;
4601         case mmMC_PMG_CMD_MRS1:
4602                 *out_reg = mmMC_SEQ_PMG_CMD_MRS1_LP;
4603                 break;
4604         case mmMC_SEQ_PMG_TIMING:
4605                 *out_reg = mmMC_SEQ_PMG_TIMING_LP;
4606                 break;
4607         case mmMC_PMG_CMD_MRS2:
4608                 *out_reg = mmMC_SEQ_PMG_CMD_MRS2_LP;
4609                 break;
4610         case mmMC_SEQ_WR_CTL_2:
4611                 *out_reg = mmMC_SEQ_WR_CTL_2_LP;
4612                 break;
4613         default:
4614                 result = false;
4615                 break;
4616         }
4617
4618         return result;
4619 }
4620
4621 static void ci_set_valid_flag(struct ci_mc_reg_table *table)
4622 {
4623         u8 i, j;
4624
4625         for (i = 0; i < table->last; i++) {
4626                 for (j = 1; j < table->num_entries; j++) {
4627                         if (table->mc_reg_table_entry[j-1].mc_data[i] !=
4628                             table->mc_reg_table_entry[j].mc_data[i]) {
4629                                 table->valid_flag |= 1 << i;
4630                                 break;
4631                         }
4632                 }
4633         }
4634 }
4635
4636 static void ci_set_s0_mc_reg_index(struct ci_mc_reg_table *table)
4637 {
4638         u32 i;
4639         u16 address;
4640
4641         for (i = 0; i < table->last; i++) {
4642                 table->mc_reg_address[i].s0 =
4643                         ci_check_s0_mc_reg_index(table->mc_reg_address[i].s1, &address) ?
4644                         address : table->mc_reg_address[i].s1;
4645         }
4646 }
4647
4648 static int ci_copy_vbios_mc_reg_table(const struct atom_mc_reg_table *table,
4649                                       struct ci_mc_reg_table *ci_table)
4650 {
4651         u8 i, j;
4652
4653         if (table->last > SMU7_DISCRETE_MC_REGISTER_ARRAY_SIZE)
4654                 return -EINVAL;
4655         if (table->num_entries > MAX_AC_TIMING_ENTRIES)
4656                 return -EINVAL;
4657
4658         for (i = 0; i < table->last; i++)
4659                 ci_table->mc_reg_address[i].s1 = table->mc_reg_address[i].s1;
4660
4661         ci_table->last = table->last;
4662
4663         for (i = 0; i < table->num_entries; i++) {
4664                 ci_table->mc_reg_table_entry[i].mclk_max =
4665                         table->mc_reg_table_entry[i].mclk_max;
4666                 for (j = 0; j < table->last; j++)
4667                         ci_table->mc_reg_table_entry[i].mc_data[j] =
4668                                 table->mc_reg_table_entry[i].mc_data[j];
4669         }
4670         ci_table->num_entries = table->num_entries;
4671
4672         return 0;
4673 }
4674
4675 static int ci_register_patching_mc_seq(struct amdgpu_device *adev,
4676                                        struct ci_mc_reg_table *table)
4677 {
4678         u8 i, k;
4679         u32 tmp;
4680         bool patch;
4681
4682         tmp = RREG32(mmMC_SEQ_MISC0);
4683         patch = ((tmp & 0x0000f00) == 0x300) ? true : false;
4684
4685         if (patch &&
4686             ((adev->pdev->device == 0x67B0) ||
4687              (adev->pdev->device == 0x67B1))) {
4688                 for (i = 0; i < table->last; i++) {
4689                         if (table->last >= SMU7_DISCRETE_MC_REGISTER_ARRAY_SIZE)
4690                                 return -EINVAL;
4691                         switch (table->mc_reg_address[i].s1) {
4692                         case mmMC_SEQ_MISC1:
4693                                 for (k = 0; k < table->num_entries; k++) {
4694                                         if ((table->mc_reg_table_entry[k].mclk_max == 125000) ||
4695                                             (table->mc_reg_table_entry[k].mclk_max == 137500))
4696                                                 table->mc_reg_table_entry[k].mc_data[i] =
4697                                                         (table->mc_reg_table_entry[k].mc_data[i] & 0xFFFFFFF8) |
4698                                                         0x00000007;
4699                                 }
4700                                 break;
4701                         case mmMC_SEQ_WR_CTL_D0:
4702                                 for (k = 0; k < table->num_entries; k++) {
4703                                         if ((table->mc_reg_table_entry[k].mclk_max == 125000) ||
4704                                             (table->mc_reg_table_entry[k].mclk_max == 137500))
4705                                                 table->mc_reg_table_entry[k].mc_data[i] =
4706                                                         (table->mc_reg_table_entry[k].mc_data[i] & 0xFFFF0F00) |
4707                                                         0x0000D0DD;
4708                                 }
4709                                 break;
4710                         case mmMC_SEQ_WR_CTL_D1:
4711                                 for (k = 0; k < table->num_entries; k++) {
4712                                         if ((table->mc_reg_table_entry[k].mclk_max == 125000) ||
4713                                             (table->mc_reg_table_entry[k].mclk_max == 137500))
4714                                                 table->mc_reg_table_entry[k].mc_data[i] =
4715                                                         (table->mc_reg_table_entry[k].mc_data[i] & 0xFFFF0F00) |
4716                                                         0x0000D0DD;
4717                                 }
4718                                 break;
4719                         case mmMC_SEQ_WR_CTL_2:
4720                                 for (k = 0; k < table->num_entries; k++) {
4721                                         if ((table->mc_reg_table_entry[k].mclk_max == 125000) ||
4722                                             (table->mc_reg_table_entry[k].mclk_max == 137500))
4723                                                 table->mc_reg_table_entry[k].mc_data[i] = 0;
4724                                 }
4725                                 break;
4726                         case mmMC_SEQ_CAS_TIMING:
4727                                 for (k = 0; k < table->num_entries; k++) {
4728                                         if (table->mc_reg_table_entry[k].mclk_max == 125000)
4729                                                 table->mc_reg_table_entry[k].mc_data[i] =
4730                                                         (table->mc_reg_table_entry[k].mc_data[i] & 0xFFE0FE0F) |
4731                                                         0x000C0140;
4732                                         else if (table->mc_reg_table_entry[k].mclk_max == 137500)
4733                                                 table->mc_reg_table_entry[k].mc_data[i] =
4734                                                         (table->mc_reg_table_entry[k].mc_data[i] & 0xFFE0FE0F) |
4735                                                         0x000C0150;
4736                                 }
4737                                 break;
4738                         case mmMC_SEQ_MISC_TIMING:
4739                                 for (k = 0; k < table->num_entries; k++) {
4740                                         if (table->mc_reg_table_entry[k].mclk_max == 125000)
4741                                                 table->mc_reg_table_entry[k].mc_data[i] =
4742                                                         (table->mc_reg_table_entry[k].mc_data[i] & 0xFFFFFFE0) |
4743                                                         0x00000030;
4744                                         else if (table->mc_reg_table_entry[k].mclk_max == 137500)
4745                                                 table->mc_reg_table_entry[k].mc_data[i] =
4746                                                         (table->mc_reg_table_entry[k].mc_data[i] & 0xFFFFFFE0) |
4747                                                         0x00000035;
4748                                 }
4749                                 break;
4750                         default:
4751                                 break;
4752                         }
4753                 }
4754
4755                 WREG32(mmMC_SEQ_IO_DEBUG_INDEX, 3);
4756                 tmp = RREG32(mmMC_SEQ_IO_DEBUG_DATA);
4757                 tmp = (tmp & 0xFFF8FFFF) | (1 << 16);
4758                 WREG32(mmMC_SEQ_IO_DEBUG_INDEX, 3);
4759                 WREG32(mmMC_SEQ_IO_DEBUG_DATA, tmp);
4760         }
4761
4762         return 0;
4763 }
4764
4765 static int ci_initialize_mc_reg_table(struct amdgpu_device *adev)
4766 {
4767         struct ci_power_info *pi = ci_get_pi(adev);
4768         struct atom_mc_reg_table *table;
4769         struct ci_mc_reg_table *ci_table = &pi->mc_reg_table;
4770         u8 module_index = ci_get_memory_module_index(adev);
4771         int ret;
4772
4773         table = kzalloc(sizeof(struct atom_mc_reg_table), GFP_KERNEL);
4774         if (!table)
4775                 return -ENOMEM;
4776
4777         WREG32(mmMC_SEQ_RAS_TIMING_LP, RREG32(mmMC_SEQ_RAS_TIMING));
4778         WREG32(mmMC_SEQ_CAS_TIMING_LP, RREG32(mmMC_SEQ_CAS_TIMING));
4779         WREG32(mmMC_SEQ_DLL_STBY_LP, RREG32(mmMC_SEQ_DLL_STBY));
4780         WREG32(mmMC_SEQ_G5PDX_CMD0_LP, RREG32(mmMC_SEQ_G5PDX_CMD0));
4781         WREG32(mmMC_SEQ_G5PDX_CMD1_LP, RREG32(mmMC_SEQ_G5PDX_CMD1));
4782         WREG32(mmMC_SEQ_G5PDX_CTRL_LP, RREG32(mmMC_SEQ_G5PDX_CTRL));
4783         WREG32(mmMC_SEQ_PMG_DVS_CMD_LP, RREG32(mmMC_SEQ_PMG_DVS_CMD));
4784         WREG32(mmMC_SEQ_PMG_DVS_CTL_LP, RREG32(mmMC_SEQ_PMG_DVS_CTL));
4785         WREG32(mmMC_SEQ_MISC_TIMING_LP, RREG32(mmMC_SEQ_MISC_TIMING));
4786         WREG32(mmMC_SEQ_MISC_TIMING2_LP, RREG32(mmMC_SEQ_MISC_TIMING2));
4787         WREG32(mmMC_SEQ_PMG_CMD_EMRS_LP, RREG32(mmMC_PMG_CMD_EMRS));
4788         WREG32(mmMC_SEQ_PMG_CMD_MRS_LP, RREG32(mmMC_PMG_CMD_MRS));
4789         WREG32(mmMC_SEQ_PMG_CMD_MRS1_LP, RREG32(mmMC_PMG_CMD_MRS1));
4790         WREG32(mmMC_SEQ_WR_CTL_D0_LP, RREG32(mmMC_SEQ_WR_CTL_D0));
4791         WREG32(mmMC_SEQ_WR_CTL_D1_LP, RREG32(mmMC_SEQ_WR_CTL_D1));
4792         WREG32(mmMC_SEQ_RD_CTL_D0_LP, RREG32(mmMC_SEQ_RD_CTL_D0));
4793         WREG32(mmMC_SEQ_RD_CTL_D1_LP, RREG32(mmMC_SEQ_RD_CTL_D1));
4794         WREG32(mmMC_SEQ_PMG_TIMING_LP, RREG32(mmMC_SEQ_PMG_TIMING));
4795         WREG32(mmMC_SEQ_PMG_CMD_MRS2_LP, RREG32(mmMC_PMG_CMD_MRS2));
4796         WREG32(mmMC_SEQ_WR_CTL_2_LP, RREG32(mmMC_SEQ_WR_CTL_2));
4797
4798         ret = amdgpu_atombios_init_mc_reg_table(adev, module_index, table);
4799         if (ret)
4800                 goto init_mc_done;
4801
4802         ret = ci_copy_vbios_mc_reg_table(table, ci_table);
4803         if (ret)
4804                 goto init_mc_done;
4805
4806         ci_set_s0_mc_reg_index(ci_table);
4807
4808         ret = ci_register_patching_mc_seq(adev, ci_table);
4809         if (ret)
4810                 goto init_mc_done;
4811
4812         ret = ci_set_mc_special_registers(adev, ci_table);
4813         if (ret)
4814                 goto init_mc_done;
4815
4816         ci_set_valid_flag(ci_table);
4817
4818 init_mc_done:
4819         kfree(table);
4820
4821         return ret;
4822 }
4823
4824 static int ci_populate_mc_reg_addresses(struct amdgpu_device *adev,
4825                                         SMU7_Discrete_MCRegisters *mc_reg_table)
4826 {
4827         struct ci_power_info *pi = ci_get_pi(adev);
4828         u32 i, j;
4829
4830         for (i = 0, j = 0; j < pi->mc_reg_table.last; j++) {
4831                 if (pi->mc_reg_table.valid_flag & (1 << j)) {
4832                         if (i >= SMU7_DISCRETE_MC_REGISTER_ARRAY_SIZE)
4833                                 return -EINVAL;
4834                         mc_reg_table->address[i].s0 = cpu_to_be16(pi->mc_reg_table.mc_reg_address[j].s0);
4835                         mc_reg_table->address[i].s1 = cpu_to_be16(pi->mc_reg_table.mc_reg_address[j].s1);
4836                         i++;
4837                 }
4838         }
4839
4840         mc_reg_table->last = (u8)i;
4841
4842         return 0;
4843 }
4844
4845 static void ci_convert_mc_registers(const struct ci_mc_reg_entry *entry,
4846                                     SMU7_Discrete_MCRegisterSet *data,
4847                                     u32 num_entries, u32 valid_flag)
4848 {
4849         u32 i, j;
4850
4851         for (i = 0, j = 0; j < num_entries; j++) {
4852                 if (valid_flag & (1 << j)) {
4853                         data->value[i] = cpu_to_be32(entry->mc_data[j]);
4854                         i++;
4855                 }
4856         }
4857 }
4858
4859 static void ci_convert_mc_reg_table_entry_to_smc(struct amdgpu_device *adev,
4860                                                  const u32 memory_clock,
4861                                                  SMU7_Discrete_MCRegisterSet *mc_reg_table_data)
4862 {
4863         struct ci_power_info *pi = ci_get_pi(adev);
4864         u32 i = 0;
4865
4866         for(i = 0; i < pi->mc_reg_table.num_entries; i++) {
4867                 if (memory_clock <= pi->mc_reg_table.mc_reg_table_entry[i].mclk_max)
4868                         break;
4869         }
4870
4871         if ((i == pi->mc_reg_table.num_entries) && (i > 0))
4872                 --i;
4873
4874         ci_convert_mc_registers(&pi->mc_reg_table.mc_reg_table_entry[i],
4875                                 mc_reg_table_data, pi->mc_reg_table.last,
4876                                 pi->mc_reg_table.valid_flag);
4877 }
4878
4879 static void ci_convert_mc_reg_table_to_smc(struct amdgpu_device *adev,
4880                                            SMU7_Discrete_MCRegisters *mc_reg_table)
4881 {
4882         struct ci_power_info *pi = ci_get_pi(adev);
4883         u32 i;
4884
4885         for (i = 0; i < pi->dpm_table.mclk_table.count; i++)
4886                 ci_convert_mc_reg_table_entry_to_smc(adev,
4887                                                      pi->dpm_table.mclk_table.dpm_levels[i].value,
4888                                                      &mc_reg_table->data[i]);
4889 }
4890
4891 static int ci_populate_initial_mc_reg_table(struct amdgpu_device *adev)
4892 {
4893         struct ci_power_info *pi = ci_get_pi(adev);
4894         int ret;
4895
4896         memset(&pi->smc_mc_reg_table, 0, sizeof(SMU7_Discrete_MCRegisters));
4897
4898         ret = ci_populate_mc_reg_addresses(adev, &pi->smc_mc_reg_table);
4899         if (ret)
4900                 return ret;
4901         ci_convert_mc_reg_table_to_smc(adev, &pi->smc_mc_reg_table);
4902
4903         return amdgpu_ci_copy_bytes_to_smc(adev,
4904                                     pi->mc_reg_table_start,
4905                                     (u8 *)&pi->smc_mc_reg_table,
4906                                     sizeof(SMU7_Discrete_MCRegisters),
4907                                     pi->sram_end);
4908 }
4909
4910 static int ci_update_and_upload_mc_reg_table(struct amdgpu_device *adev)
4911 {
4912         struct ci_power_info *pi = ci_get_pi(adev);
4913
4914         if (!(pi->need_update_smu7_dpm_table & DPMTABLE_OD_UPDATE_MCLK))
4915                 return 0;
4916
4917         memset(&pi->smc_mc_reg_table, 0, sizeof(SMU7_Discrete_MCRegisters));
4918
4919         ci_convert_mc_reg_table_to_smc(adev, &pi->smc_mc_reg_table);
4920
4921         return amdgpu_ci_copy_bytes_to_smc(adev,
4922                                     pi->mc_reg_table_start +
4923                                     offsetof(SMU7_Discrete_MCRegisters, data[0]),
4924                                     (u8 *)&pi->smc_mc_reg_table.data[0],
4925                                     sizeof(SMU7_Discrete_MCRegisterSet) *
4926                                     pi->dpm_table.mclk_table.count,
4927                                     pi->sram_end);
4928 }
4929
4930 static void ci_enable_voltage_control(struct amdgpu_device *adev)
4931 {
4932         u32 tmp = RREG32_SMC(ixGENERAL_PWRMGT);
4933
4934         tmp |= GENERAL_PWRMGT__VOLT_PWRMGT_EN_MASK;
4935         WREG32_SMC(ixGENERAL_PWRMGT, tmp);
4936 }
4937
4938 static enum amdgpu_pcie_gen ci_get_maximum_link_speed(struct amdgpu_device *adev,
4939                                                       struct amdgpu_ps *amdgpu_state)
4940 {
4941         struct ci_ps *state = ci_get_ps(amdgpu_state);
4942         int i;
4943         u16 pcie_speed, max_speed = 0;
4944
4945         for (i = 0; i < state->performance_level_count; i++) {
4946                 pcie_speed = state->performance_levels[i].pcie_gen;
4947                 if (max_speed < pcie_speed)
4948                         max_speed = pcie_speed;
4949         }
4950
4951         return max_speed;
4952 }
4953
4954 static u16 ci_get_current_pcie_speed(struct amdgpu_device *adev)
4955 {
4956         u32 speed_cntl = 0;
4957
4958         speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL) &
4959                 PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE_MASK;
4960         speed_cntl >>= PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE__SHIFT;
4961
4962         return (u16)speed_cntl;
4963 }
4964
4965 static int ci_get_current_pcie_lane_number(struct amdgpu_device *adev)
4966 {
4967         u32 link_width = 0;
4968
4969         link_width = RREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL) &
4970                 PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH_RD_MASK;
4971         link_width >>= PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH_RD__SHIFT;
4972
4973         switch (link_width) {
4974         case 1:
4975                 return 1;
4976         case 2:
4977                 return 2;
4978         case 3:
4979                 return 4;
4980         case 4:
4981                 return 8;
4982         case 0:
4983         case 6:
4984         default:
4985                 return 16;
4986         }
4987 }
4988
4989 static void ci_request_link_speed_change_before_state_change(struct amdgpu_device *adev,
4990                                                              struct amdgpu_ps *amdgpu_new_state,
4991                                                              struct amdgpu_ps *amdgpu_current_state)
4992 {
4993         struct ci_power_info *pi = ci_get_pi(adev);
4994         enum amdgpu_pcie_gen target_link_speed =
4995                 ci_get_maximum_link_speed(adev, amdgpu_new_state);
4996         enum amdgpu_pcie_gen current_link_speed;
4997
4998         if (pi->force_pcie_gen == AMDGPU_PCIE_GEN_INVALID)
4999                 current_link_speed = ci_get_maximum_link_speed(adev, amdgpu_current_state);
5000         else
5001                 current_link_speed = pi->force_pcie_gen;
5002
5003         pi->force_pcie_gen = AMDGPU_PCIE_GEN_INVALID;
5004         pi->pspp_notify_required = false;
5005         if (target_link_speed > current_link_speed) {
5006                 switch (target_link_speed) {
5007 #ifdef CONFIG_ACPI
5008                 case AMDGPU_PCIE_GEN3:
5009                         if (amdgpu_acpi_pcie_performance_request(adev, PCIE_PERF_REQ_PECI_GEN3, false) == 0)
5010                                 break;
5011                         pi->force_pcie_gen = AMDGPU_PCIE_GEN2;
5012                         if (current_link_speed == AMDGPU_PCIE_GEN2)
5013                                 break;
5014                 case AMDGPU_PCIE_GEN2:
5015                         if (amdgpu_acpi_pcie_performance_request(adev, PCIE_PERF_REQ_PECI_GEN2, false) == 0)
5016                                 break;
5017 #endif
5018                 default:
5019                         pi->force_pcie_gen = ci_get_current_pcie_speed(adev);
5020                         break;
5021                 }
5022         } else {
5023                 if (target_link_speed < current_link_speed)
5024                         pi->pspp_notify_required = true;
5025         }
5026 }
5027
5028 static void ci_notify_link_speed_change_after_state_change(struct amdgpu_device *adev,
5029                                                            struct amdgpu_ps *amdgpu_new_state,
5030                                                            struct amdgpu_ps *amdgpu_current_state)
5031 {
5032         struct ci_power_info *pi = ci_get_pi(adev);
5033         enum amdgpu_pcie_gen target_link_speed =
5034                 ci_get_maximum_link_speed(adev, amdgpu_new_state);
5035         u8 request;
5036
5037         if (pi->pspp_notify_required) {
5038                 if (target_link_speed == AMDGPU_PCIE_GEN3)
5039                         request = PCIE_PERF_REQ_PECI_GEN3;
5040                 else if (target_link_speed == AMDGPU_PCIE_GEN2)
5041                         request = PCIE_PERF_REQ_PECI_GEN2;
5042                 else
5043                         request = PCIE_PERF_REQ_PECI_GEN1;
5044
5045                 if ((request == PCIE_PERF_REQ_PECI_GEN1) &&
5046                     (ci_get_current_pcie_speed(adev) > 0))
5047                         return;
5048
5049 #ifdef CONFIG_ACPI
5050                 amdgpu_acpi_pcie_performance_request(adev, request, false);
5051 #endif
5052         }
5053 }
5054
5055 static int ci_set_private_data_variables_based_on_pptable(struct amdgpu_device *adev)
5056 {
5057         struct ci_power_info *pi = ci_get_pi(adev);
5058         struct amdgpu_clock_voltage_dependency_table *allowed_sclk_vddc_table =
5059                 &adev->pm.dpm.dyn_state.vddc_dependency_on_sclk;
5060         struct amdgpu_clock_voltage_dependency_table *allowed_mclk_vddc_table =
5061                 &adev->pm.dpm.dyn_state.vddc_dependency_on_mclk;
5062         struct amdgpu_clock_voltage_dependency_table *allowed_mclk_vddci_table =
5063                 &adev->pm.dpm.dyn_state.vddci_dependency_on_mclk;
5064
5065         if (allowed_sclk_vddc_table == NULL)
5066                 return -EINVAL;
5067         if (allowed_sclk_vddc_table->count < 1)
5068                 return -EINVAL;
5069         if (allowed_mclk_vddc_table == NULL)
5070                 return -EINVAL;
5071         if (allowed_mclk_vddc_table->count < 1)
5072                 return -EINVAL;
5073         if (allowed_mclk_vddci_table == NULL)
5074                 return -EINVAL;
5075         if (allowed_mclk_vddci_table->count < 1)
5076                 return -EINVAL;
5077
5078         pi->min_vddc_in_pp_table = allowed_sclk_vddc_table->entries[0].v;
5079         pi->max_vddc_in_pp_table =
5080                 allowed_sclk_vddc_table->entries[allowed_sclk_vddc_table->count - 1].v;
5081
5082         pi->min_vddci_in_pp_table = allowed_mclk_vddci_table->entries[0].v;
5083         pi->max_vddci_in_pp_table =
5084                 allowed_mclk_vddci_table->entries[allowed_mclk_vddci_table->count - 1].v;
5085
5086         adev->pm.dpm.dyn_state.max_clock_voltage_on_ac.sclk =
5087                 allowed_sclk_vddc_table->entries[allowed_sclk_vddc_table->count - 1].clk;
5088         adev->pm.dpm.dyn_state.max_clock_voltage_on_ac.mclk =
5089                 allowed_mclk_vddc_table->entries[allowed_sclk_vddc_table->count - 1].clk;
5090         adev->pm.dpm.dyn_state.max_clock_voltage_on_ac.vddc =
5091                 allowed_sclk_vddc_table->entries[allowed_sclk_vddc_table->count - 1].v;
5092         adev->pm.dpm.dyn_state.max_clock_voltage_on_ac.vddci =
5093                 allowed_mclk_vddci_table->entries[allowed_mclk_vddci_table->count - 1].v;
5094
5095         return 0;
5096 }
5097
5098 static void ci_patch_with_vddc_leakage(struct amdgpu_device *adev, u16 *vddc)
5099 {
5100         struct ci_power_info *pi = ci_get_pi(adev);
5101         struct ci_leakage_voltage *leakage_table = &pi->vddc_leakage;
5102         u32 leakage_index;
5103
5104         for (leakage_index = 0; leakage_index < leakage_table->count; leakage_index++) {
5105                 if (leakage_table->leakage_id[leakage_index] == *vddc) {
5106                         *vddc = leakage_table->actual_voltage[leakage_index];
5107                         break;
5108                 }
5109         }
5110 }
5111
5112 static void ci_patch_with_vddci_leakage(struct amdgpu_device *adev, u16 *vddci)
5113 {
5114         struct ci_power_info *pi = ci_get_pi(adev);
5115         struct ci_leakage_voltage *leakage_table = &pi->vddci_leakage;
5116         u32 leakage_index;
5117
5118         for (leakage_index = 0; leakage_index < leakage_table->count; leakage_index++) {
5119                 if (leakage_table->leakage_id[leakage_index] == *vddci) {
5120                         *vddci = leakage_table->actual_voltage[leakage_index];
5121                         break;
5122                 }
5123         }
5124 }
5125
5126 static void ci_patch_clock_voltage_dependency_table_with_vddc_leakage(struct amdgpu_device *adev,
5127                                                                       struct amdgpu_clock_voltage_dependency_table *table)
5128 {
5129         u32 i;
5130
5131         if (table) {
5132                 for (i = 0; i < table->count; i++)
5133                         ci_patch_with_vddc_leakage(adev, &table->entries[i].v);
5134         }
5135 }
5136
5137 static void ci_patch_clock_voltage_dependency_table_with_vddci_leakage(struct amdgpu_device *adev,
5138                                                                        struct amdgpu_clock_voltage_dependency_table *table)
5139 {
5140         u32 i;
5141
5142         if (table) {
5143                 for (i = 0; i < table->count; i++)
5144                         ci_patch_with_vddci_leakage(adev, &table->entries[i].v);
5145         }
5146 }
5147
5148 static void ci_patch_vce_clock_voltage_dependency_table_with_vddc_leakage(struct amdgpu_device *adev,
5149                                                                           struct amdgpu_vce_clock_voltage_dependency_table *table)
5150 {
5151         u32 i;
5152
5153         if (table) {
5154                 for (i = 0; i < table->count; i++)
5155                         ci_patch_with_vddc_leakage(adev, &table->entries[i].v);
5156         }
5157 }
5158
5159 static void ci_patch_uvd_clock_voltage_dependency_table_with_vddc_leakage(struct amdgpu_device *adev,
5160                                                                           struct amdgpu_uvd_clock_voltage_dependency_table *table)
5161 {
5162         u32 i;
5163
5164         if (table) {
5165                 for (i = 0; i < table->count; i++)
5166                         ci_patch_with_vddc_leakage(adev, &table->entries[i].v);
5167         }
5168 }
5169
5170 static void ci_patch_vddc_phase_shed_limit_table_with_vddc_leakage(struct amdgpu_device *adev,
5171                                                                    struct amdgpu_phase_shedding_limits_table *table)
5172 {
5173         u32 i;
5174
5175         if (table) {
5176                 for (i = 0; i < table->count; i++)
5177                         ci_patch_with_vddc_leakage(adev, &table->entries[i].voltage);
5178         }
5179 }
5180
5181 static void ci_patch_clock_voltage_limits_with_vddc_leakage(struct amdgpu_device *adev,
5182                                                             struct amdgpu_clock_and_voltage_limits *table)
5183 {
5184         if (table) {
5185                 ci_patch_with_vddc_leakage(adev, (u16 *)&table->vddc);
5186                 ci_patch_with_vddci_leakage(adev, (u16 *)&table->vddci);
5187         }
5188 }
5189
5190 static void ci_patch_cac_leakage_table_with_vddc_leakage(struct amdgpu_device *adev,
5191                                                          struct amdgpu_cac_leakage_table *table)
5192 {
5193         u32 i;
5194
5195         if (table) {
5196                 for (i = 0; i < table->count; i++)
5197                         ci_patch_with_vddc_leakage(adev, &table->entries[i].vddc);
5198         }
5199 }
5200
5201 static void ci_patch_dependency_tables_with_leakage(struct amdgpu_device *adev)
5202 {
5203
5204         ci_patch_clock_voltage_dependency_table_with_vddc_leakage(adev,
5205                                                                   &adev->pm.dpm.dyn_state.vddc_dependency_on_sclk);
5206         ci_patch_clock_voltage_dependency_table_with_vddc_leakage(adev,
5207                                                                   &adev->pm.dpm.dyn_state.vddc_dependency_on_mclk);
5208         ci_patch_clock_voltage_dependency_table_with_vddc_leakage(adev,
5209                                                                   &adev->pm.dpm.dyn_state.vddc_dependency_on_dispclk);
5210         ci_patch_clock_voltage_dependency_table_with_vddci_leakage(adev,
5211                                                                    &adev->pm.dpm.dyn_state.vddci_dependency_on_mclk);
5212         ci_patch_vce_clock_voltage_dependency_table_with_vddc_leakage(adev,
5213                                                                       &adev->pm.dpm.dyn_state.vce_clock_voltage_dependency_table);
5214         ci_patch_uvd_clock_voltage_dependency_table_with_vddc_leakage(adev,
5215                                                                       &adev->pm.dpm.dyn_state.uvd_clock_voltage_dependency_table);
5216         ci_patch_clock_voltage_dependency_table_with_vddc_leakage(adev,
5217                                                                   &adev->pm.dpm.dyn_state.samu_clock_voltage_dependency_table);
5218         ci_patch_clock_voltage_dependency_table_with_vddc_leakage(adev,
5219                                                                   &adev->pm.dpm.dyn_state.acp_clock_voltage_dependency_table);
5220         ci_patch_vddc_phase_shed_limit_table_with_vddc_leakage(adev,
5221                                                                &adev->pm.dpm.dyn_state.phase_shedding_limits_table);
5222         ci_patch_clock_voltage_limits_with_vddc_leakage(adev,
5223                                                         &adev->pm.dpm.dyn_state.max_clock_voltage_on_ac);
5224         ci_patch_clock_voltage_limits_with_vddc_leakage(adev,
5225                                                         &adev->pm.dpm.dyn_state.max_clock_voltage_on_dc);
5226         ci_patch_cac_leakage_table_with_vddc_leakage(adev,
5227                                                      &adev->pm.dpm.dyn_state.cac_leakage_table);
5228
5229 }
5230
5231 static void ci_update_current_ps(struct amdgpu_device *adev,
5232                                  struct amdgpu_ps *rps)
5233 {
5234         struct ci_ps *new_ps = ci_get_ps(rps);
5235         struct ci_power_info *pi = ci_get_pi(adev);
5236
5237         pi->current_rps = *rps;
5238         pi->current_ps = *new_ps;
5239         pi->current_rps.ps_priv = &pi->current_ps;
5240 }
5241
5242 static void ci_update_requested_ps(struct amdgpu_device *adev,
5243                                    struct amdgpu_ps *rps)
5244 {
5245         struct ci_ps *new_ps = ci_get_ps(rps);
5246         struct ci_power_info *pi = ci_get_pi(adev);
5247
5248         pi->requested_rps = *rps;
5249         pi->requested_ps = *new_ps;
5250         pi->requested_rps.ps_priv = &pi->requested_ps;
5251 }
5252
5253 static int ci_dpm_pre_set_power_state(struct amdgpu_device *adev)
5254 {
5255         struct ci_power_info *pi = ci_get_pi(adev);
5256         struct amdgpu_ps requested_ps = *adev->pm.dpm.requested_ps;
5257         struct amdgpu_ps *new_ps = &requested_ps;
5258
5259         ci_update_requested_ps(adev, new_ps);
5260
5261         ci_apply_state_adjust_rules(adev, &pi->requested_rps);
5262
5263         return 0;
5264 }
5265
5266 static void ci_dpm_post_set_power_state(struct amdgpu_device *adev)
5267 {
5268         struct ci_power_info *pi = ci_get_pi(adev);
5269         struct amdgpu_ps *new_ps = &pi->requested_rps;
5270
5271         ci_update_current_ps(adev, new_ps);
5272 }
5273
5274
5275 static void ci_dpm_setup_asic(struct amdgpu_device *adev)
5276 {
5277         ci_read_clock_registers(adev);
5278         ci_enable_acpi_power_management(adev);
5279         ci_init_sclk_t(adev);
5280 }
5281
5282 static int ci_dpm_enable(struct amdgpu_device *adev)
5283 {
5284         struct ci_power_info *pi = ci_get_pi(adev);
5285         struct amdgpu_ps *boot_ps = adev->pm.dpm.boot_ps;
5286         int ret;
5287
5288         if (amdgpu_ci_is_smc_running(adev))
5289                 return -EINVAL;
5290         if (pi->voltage_control != CISLANDS_VOLTAGE_CONTROL_NONE) {
5291                 ci_enable_voltage_control(adev);
5292                 ret = ci_construct_voltage_tables(adev);
5293                 if (ret) {
5294                         DRM_ERROR("ci_construct_voltage_tables failed\n");
5295                         return ret;
5296                 }
5297         }
5298         if (pi->caps_dynamic_ac_timing) {
5299                 ret = ci_initialize_mc_reg_table(adev);
5300                 if (ret)
5301                         pi->caps_dynamic_ac_timing = false;
5302         }
5303         if (pi->dynamic_ss)
5304                 ci_enable_spread_spectrum(adev, true);
5305         if (pi->thermal_protection)
5306                 ci_enable_thermal_protection(adev, true);
5307         ci_program_sstp(adev);
5308         ci_enable_display_gap(adev);
5309         ci_program_vc(adev);
5310         ret = ci_upload_firmware(adev);
5311         if (ret) {
5312                 DRM_ERROR("ci_upload_firmware failed\n");
5313                 return ret;
5314         }
5315         ret = ci_process_firmware_header(adev);
5316         if (ret) {
5317                 DRM_ERROR("ci_process_firmware_header failed\n");
5318                 return ret;
5319         }
5320         ret = ci_initial_switch_from_arb_f0_to_f1(adev);
5321         if (ret) {
5322                 DRM_ERROR("ci_initial_switch_from_arb_f0_to_f1 failed\n");
5323                 return ret;
5324         }
5325         ret = ci_init_smc_table(adev);
5326         if (ret) {
5327                 DRM_ERROR("ci_init_smc_table failed\n");
5328                 return ret;
5329         }
5330         ret = ci_init_arb_table_index(adev);
5331         if (ret) {
5332                 DRM_ERROR("ci_init_arb_table_index failed\n");
5333                 return ret;
5334         }
5335         if (pi->caps_dynamic_ac_timing) {
5336                 ret = ci_populate_initial_mc_reg_table(adev);
5337                 if (ret) {
5338                         DRM_ERROR("ci_populate_initial_mc_reg_table failed\n");
5339                         return ret;
5340                 }
5341         }
5342         ret = ci_populate_pm_base(adev);
5343         if (ret) {
5344                 DRM_ERROR("ci_populate_pm_base failed\n");
5345                 return ret;
5346         }
5347         ci_dpm_start_smc(adev);
5348         ci_enable_vr_hot_gpio_interrupt(adev);
5349         ret = ci_notify_smc_display_change(adev, false);
5350         if (ret) {
5351                 DRM_ERROR("ci_notify_smc_display_change failed\n");
5352                 return ret;
5353         }
5354         ci_enable_sclk_control(adev, true);
5355         ret = ci_enable_ulv(adev, true);
5356         if (ret) {
5357                 DRM_ERROR("ci_enable_ulv failed\n");
5358                 return ret;
5359         }
5360         ret = ci_enable_ds_master_switch(adev, true);
5361         if (ret) {
5362                 DRM_ERROR("ci_enable_ds_master_switch failed\n");
5363                 return ret;
5364         }
5365         ret = ci_start_dpm(adev);
5366         if (ret) {
5367                 DRM_ERROR("ci_start_dpm failed\n");
5368                 return ret;
5369         }
5370         ret = ci_enable_didt(adev, true);
5371         if (ret) {
5372                 DRM_ERROR("ci_enable_didt failed\n");
5373                 return ret;
5374         }
5375         ret = ci_enable_smc_cac(adev, true);
5376         if (ret) {
5377                 DRM_ERROR("ci_enable_smc_cac failed\n");
5378                 return ret;
5379         }
5380         ret = ci_enable_power_containment(adev, true);
5381         if (ret) {
5382                 DRM_ERROR("ci_enable_power_containment failed\n");
5383                 return ret;
5384         }
5385
5386         ret = ci_power_control_set_level(adev);
5387         if (ret) {
5388                 DRM_ERROR("ci_power_control_set_level failed\n");
5389                 return ret;
5390         }
5391
5392         ci_enable_auto_throttle_source(adev, AMDGPU_DPM_AUTO_THROTTLE_SRC_THERMAL, true);
5393
5394         ret = ci_enable_thermal_based_sclk_dpm(adev, true);
5395         if (ret) {
5396                 DRM_ERROR("ci_enable_thermal_based_sclk_dpm failed\n");
5397                 return ret;
5398         }
5399
5400         ci_thermal_start_thermal_controller(adev);
5401
5402         ci_update_current_ps(adev, boot_ps);
5403
5404         if (adev->irq.installed &&
5405             amdgpu_is_internal_thermal_sensor(adev->pm.int_thermal_type)) {
5406 #if 0
5407                 PPSMC_Result result;
5408 #endif
5409                 ret = ci_thermal_set_temperature_range(adev, CISLANDS_TEMP_RANGE_MIN,
5410                                                        CISLANDS_TEMP_RANGE_MAX);
5411                 if (ret) {
5412                         DRM_ERROR("ci_thermal_set_temperature_range failed\n");
5413                         return ret;
5414                 }
5415                 amdgpu_irq_get(adev, &adev->pm.dpm.thermal.irq,
5416                                AMDGPU_THERMAL_IRQ_LOW_TO_HIGH);
5417                 amdgpu_irq_get(adev, &adev->pm.dpm.thermal.irq,
5418                                AMDGPU_THERMAL_IRQ_HIGH_TO_LOW);
5419
5420 #if 0
5421                 result = amdgpu_ci_send_msg_to_smc(adev, PPSMC_MSG_EnableThermalInterrupt);
5422
5423                 if (result != PPSMC_Result_OK)
5424                         DRM_DEBUG_KMS("Could not enable thermal interrupts.\n");
5425 #endif
5426         }
5427
5428         return 0;
5429 }
5430
5431 static void ci_dpm_disable(struct amdgpu_device *adev)
5432 {
5433         struct ci_power_info *pi = ci_get_pi(adev);
5434         struct amdgpu_ps *boot_ps = adev->pm.dpm.boot_ps;
5435
5436         amdgpu_irq_put(adev, &adev->pm.dpm.thermal.irq,
5437                        AMDGPU_THERMAL_IRQ_LOW_TO_HIGH);
5438         amdgpu_irq_put(adev, &adev->pm.dpm.thermal.irq,
5439                        AMDGPU_THERMAL_IRQ_HIGH_TO_LOW);
5440
5441         ci_dpm_powergate_uvd(adev, false);
5442
5443         if (!amdgpu_ci_is_smc_running(adev))
5444                 return;
5445
5446         ci_thermal_stop_thermal_controller(adev);
5447
5448         if (pi->thermal_protection)
5449                 ci_enable_thermal_protection(adev, false);
5450         ci_enable_power_containment(adev, false);
5451         ci_enable_smc_cac(adev, false);
5452         ci_enable_didt(adev, false);
5453         ci_enable_spread_spectrum(adev, false);
5454         ci_enable_auto_throttle_source(adev, AMDGPU_DPM_AUTO_THROTTLE_SRC_THERMAL, false);
5455         ci_stop_dpm(adev);
5456         ci_enable_ds_master_switch(adev, false);
5457         ci_enable_ulv(adev, false);
5458         ci_clear_vc(adev);
5459         ci_reset_to_default(adev);
5460         ci_dpm_stop_smc(adev);
5461         ci_force_switch_to_arb_f0(adev);
5462         ci_enable_thermal_based_sclk_dpm(adev, false);
5463
5464         ci_update_current_ps(adev, boot_ps);
5465 }
5466
5467 static int ci_dpm_set_power_state(struct amdgpu_device *adev)
5468 {
5469         struct ci_power_info *pi = ci_get_pi(adev);
5470         struct amdgpu_ps *new_ps = &pi->requested_rps;
5471         struct amdgpu_ps *old_ps = &pi->current_rps;
5472         int ret;
5473
5474         ci_find_dpm_states_clocks_in_dpm_table(adev, new_ps);
5475         if (pi->pcie_performance_request)
5476                 ci_request_link_speed_change_before_state_change(adev, new_ps, old_ps);
5477         ret = ci_freeze_sclk_mclk_dpm(adev);
5478         if (ret) {
5479                 DRM_ERROR("ci_freeze_sclk_mclk_dpm failed\n");
5480                 return ret;
5481         }
5482         ret = ci_populate_and_upload_sclk_mclk_dpm_levels(adev, new_ps);
5483         if (ret) {
5484                 DRM_ERROR("ci_populate_and_upload_sclk_mclk_dpm_levels failed\n");
5485                 return ret;
5486         }
5487         ret = ci_generate_dpm_level_enable_mask(adev, new_ps);
5488         if (ret) {
5489                 DRM_ERROR("ci_generate_dpm_level_enable_mask failed\n");
5490                 return ret;
5491         }
5492
5493         ret = ci_update_vce_dpm(adev, new_ps, old_ps);
5494         if (ret) {
5495                 DRM_ERROR("ci_update_vce_dpm failed\n");
5496                 return ret;
5497         }
5498
5499         ret = ci_update_sclk_t(adev);
5500         if (ret) {
5501                 DRM_ERROR("ci_update_sclk_t failed\n");
5502                 return ret;
5503         }
5504         if (pi->caps_dynamic_ac_timing) {
5505                 ret = ci_update_and_upload_mc_reg_table(adev);
5506                 if (ret) {
5507                         DRM_ERROR("ci_update_and_upload_mc_reg_table failed\n");
5508                         return ret;
5509                 }
5510         }
5511         ret = ci_program_memory_timing_parameters(adev);
5512         if (ret) {
5513                 DRM_ERROR("ci_program_memory_timing_parameters failed\n");
5514                 return ret;
5515         }
5516         ret = ci_unfreeze_sclk_mclk_dpm(adev);
5517         if (ret) {
5518                 DRM_ERROR("ci_unfreeze_sclk_mclk_dpm failed\n");
5519                 return ret;
5520         }
5521         ret = ci_upload_dpm_level_enable_mask(adev);
5522         if (ret) {
5523                 DRM_ERROR("ci_upload_dpm_level_enable_mask failed\n");
5524                 return ret;
5525         }
5526         if (pi->pcie_performance_request)
5527                 ci_notify_link_speed_change_after_state_change(adev, new_ps, old_ps);
5528
5529         return 0;
5530 }
5531
5532 #if 0
5533 static void ci_dpm_reset_asic(struct amdgpu_device *adev)
5534 {
5535         ci_set_boot_state(adev);
5536 }
5537 #endif
5538
5539 static void ci_dpm_display_configuration_changed(struct amdgpu_device *adev)
5540 {
5541         ci_program_display_gap(adev);
5542 }
5543
5544 union power_info {
5545         struct _ATOM_POWERPLAY_INFO info;
5546         struct _ATOM_POWERPLAY_INFO_V2 info_2;
5547         struct _ATOM_POWERPLAY_INFO_V3 info_3;
5548         struct _ATOM_PPLIB_POWERPLAYTABLE pplib;
5549         struct _ATOM_PPLIB_POWERPLAYTABLE2 pplib2;
5550         struct _ATOM_PPLIB_POWERPLAYTABLE3 pplib3;
5551 };
5552
5553 union pplib_clock_info {
5554         struct _ATOM_PPLIB_R600_CLOCK_INFO r600;
5555         struct _ATOM_PPLIB_RS780_CLOCK_INFO rs780;
5556         struct _ATOM_PPLIB_EVERGREEN_CLOCK_INFO evergreen;
5557         struct _ATOM_PPLIB_SUMO_CLOCK_INFO sumo;
5558         struct _ATOM_PPLIB_SI_CLOCK_INFO si;
5559         struct _ATOM_PPLIB_CI_CLOCK_INFO ci;
5560 };
5561
5562 union pplib_power_state {
5563         struct _ATOM_PPLIB_STATE v1;
5564         struct _ATOM_PPLIB_STATE_V2 v2;
5565 };
5566
5567 static void ci_parse_pplib_non_clock_info(struct amdgpu_device *adev,
5568                                           struct amdgpu_ps *rps,
5569                                           struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info,
5570                                           u8 table_rev)
5571 {
5572         rps->caps = le32_to_cpu(non_clock_info->ulCapsAndSettings);
5573         rps->class = le16_to_cpu(non_clock_info->usClassification);
5574         rps->class2 = le16_to_cpu(non_clock_info->usClassification2);
5575
5576         if (ATOM_PPLIB_NONCLOCKINFO_VER1 < table_rev) {
5577                 rps->vclk = le32_to_cpu(non_clock_info->ulVCLK);
5578                 rps->dclk = le32_to_cpu(non_clock_info->ulDCLK);
5579         } else {
5580                 rps->vclk = 0;
5581                 rps->dclk = 0;
5582         }
5583
5584         if (rps->class & ATOM_PPLIB_CLASSIFICATION_BOOT)
5585                 adev->pm.dpm.boot_ps = rps;
5586         if (rps->class & ATOM_PPLIB_CLASSIFICATION_UVDSTATE)
5587                 adev->pm.dpm.uvd_ps = rps;
5588 }
5589
5590 static void ci_parse_pplib_clock_info(struct amdgpu_device *adev,
5591                                       struct amdgpu_ps *rps, int index,
5592                                       union pplib_clock_info *clock_info)
5593 {
5594         struct ci_power_info *pi = ci_get_pi(adev);
5595         struct ci_ps *ps = ci_get_ps(rps);
5596         struct ci_pl *pl = &ps->performance_levels[index];
5597
5598         ps->performance_level_count = index + 1;
5599
5600         pl->sclk = le16_to_cpu(clock_info->ci.usEngineClockLow);
5601         pl->sclk |= clock_info->ci.ucEngineClockHigh << 16;
5602         pl->mclk = le16_to_cpu(clock_info->ci.usMemoryClockLow);
5603         pl->mclk |= clock_info->ci.ucMemoryClockHigh << 16;
5604
5605         pl->pcie_gen = amdgpu_get_pcie_gen_support(adev,
5606                                                    pi->sys_pcie_mask,
5607                                                    pi->vbios_boot_state.pcie_gen_bootup_value,
5608                                                    clock_info->ci.ucPCIEGen);
5609         pl->pcie_lane = amdgpu_get_pcie_lane_support(adev,
5610                                                      pi->vbios_boot_state.pcie_lane_bootup_value,
5611                                                      le16_to_cpu(clock_info->ci.usPCIELane));
5612
5613         if (rps->class & ATOM_PPLIB_CLASSIFICATION_ACPI) {
5614                 pi->acpi_pcie_gen = pl->pcie_gen;
5615         }
5616
5617         if (rps->class2 & ATOM_PPLIB_CLASSIFICATION2_ULV) {
5618                 pi->ulv.supported = true;
5619                 pi->ulv.pl = *pl;
5620                 pi->ulv.cg_ulv_parameter = CISLANDS_CGULVPARAMETER_DFLT;
5621         }
5622
5623         /* patch up boot state */
5624         if (rps->class & ATOM_PPLIB_CLASSIFICATION_BOOT) {
5625                 pl->mclk = pi->vbios_boot_state.mclk_bootup_value;
5626                 pl->sclk = pi->vbios_boot_state.sclk_bootup_value;
5627                 pl->pcie_gen = pi->vbios_boot_state.pcie_gen_bootup_value;
5628                 pl->pcie_lane = pi->vbios_boot_state.pcie_lane_bootup_value;
5629         }
5630
5631         switch (rps->class & ATOM_PPLIB_CLASSIFICATION_UI_MASK) {
5632         case ATOM_PPLIB_CLASSIFICATION_UI_BATTERY:
5633                 pi->use_pcie_powersaving_levels = true;
5634                 if (pi->pcie_gen_powersaving.max < pl->pcie_gen)
5635                         pi->pcie_gen_powersaving.max = pl->pcie_gen;
5636                 if (pi->pcie_gen_powersaving.min > pl->pcie_gen)
5637                         pi->pcie_gen_powersaving.min = pl->pcie_gen;
5638                 if (pi->pcie_lane_powersaving.max < pl->pcie_lane)
5639                         pi->pcie_lane_powersaving.max = pl->pcie_lane;
5640                 if (pi->pcie_lane_powersaving.min > pl->pcie_lane)
5641                         pi->pcie_lane_powersaving.min = pl->pcie_lane;
5642                 break;
5643         case ATOM_PPLIB_CLASSIFICATION_UI_PERFORMANCE:
5644                 pi->use_pcie_performance_levels = true;
5645                 if (pi->pcie_gen_performance.max < pl->pcie_gen)
5646                         pi->pcie_gen_performance.max = pl->pcie_gen;
5647                 if (pi->pcie_gen_performance.min > pl->pcie_gen)
5648                         pi->pcie_gen_performance.min = pl->pcie_gen;
5649                 if (pi->pcie_lane_performance.max < pl->pcie_lane)
5650                         pi->pcie_lane_performance.max = pl->pcie_lane;
5651                 if (pi->pcie_lane_performance.min > pl->pcie_lane)
5652                         pi->pcie_lane_performance.min = pl->pcie_lane;
5653                 break;
5654         default:
5655                 break;
5656         }
5657 }
5658
5659 static int ci_parse_power_table(struct amdgpu_device *adev)
5660 {
5661         struct amdgpu_mode_info *mode_info = &adev->mode_info;
5662         struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info;
5663         union pplib_power_state *power_state;
5664         int i, j, k, non_clock_array_index, clock_array_index;
5665         union pplib_clock_info *clock_info;
5666         struct _StateArray *state_array;
5667         struct _ClockInfoArray *clock_info_array;
5668         struct _NonClockInfoArray *non_clock_info_array;
5669         union power_info *power_info;
5670         int index = GetIndexIntoMasterTable(DATA, PowerPlayInfo);
5671         u16 data_offset;
5672         u8 frev, crev;
5673         u8 *power_state_offset;
5674         struct ci_ps *ps;
5675
5676         if (!amdgpu_atom_parse_data_header(mode_info->atom_context, index, NULL,
5677                                    &frev, &crev, &data_offset))
5678                 return -EINVAL;
5679         power_info = (union power_info *)(mode_info->atom_context->bios + data_offset);
5680
5681         amdgpu_add_thermal_controller(adev);
5682
5683         state_array = (struct _StateArray *)
5684                 (mode_info->atom_context->bios + data_offset +
5685                  le16_to_cpu(power_info->pplib.usStateArrayOffset));
5686         clock_info_array = (struct _ClockInfoArray *)
5687                 (mode_info->atom_context->bios + data_offset +
5688                  le16_to_cpu(power_info->pplib.usClockInfoArrayOffset));
5689         non_clock_info_array = (struct _NonClockInfoArray *)
5690                 (mode_info->atom_context->bios + data_offset +
5691                  le16_to_cpu(power_info->pplib.usNonClockInfoArrayOffset));
5692
5693         adev->pm.dpm.ps = kzalloc(sizeof(struct amdgpu_ps) *
5694                                   state_array->ucNumEntries, GFP_KERNEL);
5695         if (!adev->pm.dpm.ps)
5696                 return -ENOMEM;
5697         power_state_offset = (u8 *)state_array->states;
5698         for (i = 0; i < state_array->ucNumEntries; i++) {
5699                 u8 *idx;
5700                 power_state = (union pplib_power_state *)power_state_offset;
5701                 non_clock_array_index = power_state->v2.nonClockInfoIndex;
5702                 non_clock_info = (struct _ATOM_PPLIB_NONCLOCK_INFO *)
5703                         &non_clock_info_array->nonClockInfo[non_clock_array_index];
5704                 ps = kzalloc(sizeof(struct ci_ps), GFP_KERNEL);
5705                 if (ps == NULL) {
5706                         kfree(adev->pm.dpm.ps);
5707                         return -ENOMEM;
5708                 }
5709                 adev->pm.dpm.ps[i].ps_priv = ps;
5710                 ci_parse_pplib_non_clock_info(adev, &adev->pm.dpm.ps[i],
5711                                               non_clock_info,
5712                                               non_clock_info_array->ucEntrySize);
5713                 k = 0;
5714                 idx = (u8 *)&power_state->v2.clockInfoIndex[0];
5715                 for (j = 0; j < power_state->v2.ucNumDPMLevels; j++) {
5716                         clock_array_index = idx[j];
5717                         if (clock_array_index >= clock_info_array->ucNumEntries)
5718                                 continue;
5719                         if (k >= CISLANDS_MAX_HARDWARE_POWERLEVELS)
5720                                 break;
5721                         clock_info = (union pplib_clock_info *)
5722                                 ((u8 *)&clock_info_array->clockInfo[0] +
5723                                  (clock_array_index * clock_info_array->ucEntrySize));
5724                         ci_parse_pplib_clock_info(adev,
5725                                                   &adev->pm.dpm.ps[i], k,
5726                                                   clock_info);
5727                         k++;
5728                 }
5729                 power_state_offset += 2 + power_state->v2.ucNumDPMLevels;
5730         }
5731         adev->pm.dpm.num_ps = state_array->ucNumEntries;
5732
5733         /* fill in the vce power states */
5734         for (i = 0; i < AMDGPU_MAX_VCE_LEVELS; i++) {
5735                 u32 sclk, mclk;
5736                 clock_array_index = adev->pm.dpm.vce_states[i].clk_idx;
5737                 clock_info = (union pplib_clock_info *)
5738                         &clock_info_array->clockInfo[clock_array_index * clock_info_array->ucEntrySize];
5739                 sclk = le16_to_cpu(clock_info->ci.usEngineClockLow);
5740                 sclk |= clock_info->ci.ucEngineClockHigh << 16;
5741                 mclk = le16_to_cpu(clock_info->ci.usMemoryClockLow);
5742                 mclk |= clock_info->ci.ucMemoryClockHigh << 16;
5743                 adev->pm.dpm.vce_states[i].sclk = sclk;
5744                 adev->pm.dpm.vce_states[i].mclk = mclk;
5745         }
5746
5747         return 0;
5748 }
5749
5750 static int ci_get_vbios_boot_values(struct amdgpu_device *adev,
5751                                     struct ci_vbios_boot_state *boot_state)
5752 {
5753         struct amdgpu_mode_info *mode_info = &adev->mode_info;
5754         int index = GetIndexIntoMasterTable(DATA, FirmwareInfo);
5755         ATOM_FIRMWARE_INFO_V2_2 *firmware_info;
5756         u8 frev, crev;
5757         u16 data_offset;
5758
5759         if (amdgpu_atom_parse_data_header(mode_info->atom_context, index, NULL,
5760                                    &frev, &crev, &data_offset)) {
5761                 firmware_info =
5762                         (ATOM_FIRMWARE_INFO_V2_2 *)(mode_info->atom_context->bios +
5763                                                     data_offset);
5764                 boot_state->mvdd_bootup_value = le16_to_cpu(firmware_info->usBootUpMVDDCVoltage);
5765                 boot_state->vddc_bootup_value = le16_to_cpu(firmware_info->usBootUpVDDCVoltage);
5766                 boot_state->vddci_bootup_value = le16_to_cpu(firmware_info->usBootUpVDDCIVoltage);
5767                 boot_state->pcie_gen_bootup_value = ci_get_current_pcie_speed(adev);
5768                 boot_state->pcie_lane_bootup_value = ci_get_current_pcie_lane_number(adev);
5769                 boot_state->sclk_bootup_value = le32_to_cpu(firmware_info->ulDefaultEngineClock);
5770                 boot_state->mclk_bootup_value = le32_to_cpu(firmware_info->ulDefaultMemoryClock);
5771
5772                 return 0;
5773         }
5774         return -EINVAL;
5775 }
5776
5777 static void ci_dpm_fini(struct amdgpu_device *adev)
5778 {
5779         int i;
5780
5781         for (i = 0; i < adev->pm.dpm.num_ps; i++) {
5782                 kfree(adev->pm.dpm.ps[i].ps_priv);
5783         }
5784         kfree(adev->pm.dpm.ps);
5785         kfree(adev->pm.dpm.priv);
5786         kfree(adev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries);
5787         amdgpu_free_extended_power_table(adev);
5788 }
5789
5790 /**
5791  * ci_dpm_init_microcode - load ucode images from disk
5792  *
5793  * @adev: amdgpu_device pointer
5794  *
5795  * Use the firmware interface to load the ucode images into
5796  * the driver (not loaded into hw).
5797  * Returns 0 on success, error on failure.
5798  */
5799 static int ci_dpm_init_microcode(struct amdgpu_device *adev)
5800 {
5801         const char *chip_name;
5802         char fw_name[30];
5803         int err;
5804
5805         DRM_DEBUG("\n");
5806
5807         switch (adev->asic_type) {
5808         case CHIP_BONAIRE:
5809                 chip_name = "bonaire";
5810                 break;
5811         case CHIP_HAWAII:
5812                 chip_name = "hawaii";
5813                 break;
5814         case CHIP_KAVERI:
5815         case CHIP_KABINI:
5816         default: BUG();
5817         }
5818
5819         snprintf(fw_name, sizeof(fw_name), "/*(DEBLOBBED)*/", chip_name);
5820         err = reject_firmware(&adev->pm.fw, fw_name, adev->dev);
5821         if (err)
5822                 goto out;
5823         err = amdgpu_ucode_validate(adev->pm.fw);
5824
5825 out:
5826         if (err) {
5827                 printk(KERN_ERR
5828                        "cik_smc: Failed to load firmware \"%s\"\n",
5829                        fw_name);
5830                 release_firmware(adev->pm.fw);
5831                 adev->pm.fw = NULL;
5832         }
5833         return err;
5834 }
5835
5836 static int ci_dpm_init(struct amdgpu_device *adev)
5837 {
5838         int index = GetIndexIntoMasterTable(DATA, ASIC_InternalSS_Info);
5839         SMU7_Discrete_DpmTable *dpm_table;
5840         struct amdgpu_gpio_rec gpio;
5841         u16 data_offset, size;
5842         u8 frev, crev;
5843         struct ci_power_info *pi;
5844         int ret;
5845         u32 mask;
5846
5847         pi = kzalloc(sizeof(struct ci_power_info), GFP_KERNEL);
5848         if (pi == NULL)
5849                 return -ENOMEM;
5850         adev->pm.dpm.priv = pi;
5851
5852         ret = drm_pcie_get_speed_cap_mask(adev->ddev, &mask);
5853         if (ret)
5854                 pi->sys_pcie_mask = 0;
5855         else
5856                 pi->sys_pcie_mask = mask;
5857         pi->force_pcie_gen = AMDGPU_PCIE_GEN_INVALID;
5858
5859         pi->pcie_gen_performance.max = AMDGPU_PCIE_GEN1;
5860         pi->pcie_gen_performance.min = AMDGPU_PCIE_GEN3;
5861         pi->pcie_gen_powersaving.max = AMDGPU_PCIE_GEN1;
5862         pi->pcie_gen_powersaving.min = AMDGPU_PCIE_GEN3;
5863
5864         pi->pcie_lane_performance.max = 0;
5865         pi->pcie_lane_performance.min = 16;
5866         pi->pcie_lane_powersaving.max = 0;
5867         pi->pcie_lane_powersaving.min = 16;
5868
5869         ret = ci_get_vbios_boot_values(adev, &pi->vbios_boot_state);
5870         if (ret) {
5871                 ci_dpm_fini(adev);
5872                 return ret;
5873         }
5874
5875         ret = amdgpu_get_platform_caps(adev);
5876         if (ret) {
5877                 ci_dpm_fini(adev);
5878                 return ret;
5879         }
5880
5881         ret = amdgpu_parse_extended_power_table(adev);
5882         if (ret) {
5883                 ci_dpm_fini(adev);
5884                 return ret;
5885         }
5886
5887         ret = ci_parse_power_table(adev);
5888         if (ret) {
5889                 ci_dpm_fini(adev);
5890                 return ret;
5891         }
5892
5893         pi->dll_default_on = false;
5894         pi->sram_end = SMC_RAM_END;
5895
5896         pi->activity_target[0] = CISLAND_TARGETACTIVITY_DFLT;
5897         pi->activity_target[1] = CISLAND_TARGETACTIVITY_DFLT;
5898         pi->activity_target[2] = CISLAND_TARGETACTIVITY_DFLT;
5899         pi->activity_target[3] = CISLAND_TARGETACTIVITY_DFLT;
5900         pi->activity_target[4] = CISLAND_TARGETACTIVITY_DFLT;
5901         pi->activity_target[5] = CISLAND_TARGETACTIVITY_DFLT;
5902         pi->activity_target[6] = CISLAND_TARGETACTIVITY_DFLT;
5903         pi->activity_target[7] = CISLAND_TARGETACTIVITY_DFLT;
5904
5905         pi->mclk_activity_target = CISLAND_MCLK_TARGETACTIVITY_DFLT;
5906
5907         pi->sclk_dpm_key_disabled = 0;
5908         pi->mclk_dpm_key_disabled = 0;
5909         pi->pcie_dpm_key_disabled = 0;
5910         pi->thermal_sclk_dpm_enabled = 0;
5911
5912         pi->caps_sclk_ds = true;
5913
5914         pi->mclk_strobe_mode_threshold = 40000;
5915         pi->mclk_stutter_mode_threshold = 40000;
5916         pi->mclk_edc_enable_threshold = 40000;
5917         pi->mclk_edc_wr_enable_threshold = 40000;
5918
5919         ci_initialize_powertune_defaults(adev);
5920
5921         pi->caps_fps = false;
5922
5923         pi->caps_sclk_throttle_low_notification = false;
5924
5925         pi->caps_uvd_dpm = true;
5926         pi->caps_vce_dpm = true;
5927
5928         ci_get_leakage_voltages(adev);
5929         ci_patch_dependency_tables_with_leakage(adev);
5930         ci_set_private_data_variables_based_on_pptable(adev);
5931
5932         adev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries =
5933                 kzalloc(4 * sizeof(struct amdgpu_clock_voltage_dependency_entry), GFP_KERNEL);
5934         if (!adev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries) {
5935                 ci_dpm_fini(adev);
5936                 return -ENOMEM;
5937         }
5938         adev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.count = 4;
5939         adev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[0].clk = 0;
5940         adev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[0].v = 0;
5941         adev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[1].clk = 36000;
5942         adev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[1].v = 720;
5943         adev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[2].clk = 54000;
5944         adev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[2].v = 810;
5945         adev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[3].clk = 72000;
5946         adev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[3].v = 900;
5947
5948         adev->pm.dpm.dyn_state.mclk_sclk_ratio = 4;
5949         adev->pm.dpm.dyn_state.sclk_mclk_delta = 15000;
5950         adev->pm.dpm.dyn_state.vddc_vddci_delta = 200;
5951
5952         adev->pm.dpm.dyn_state.valid_sclk_values.count = 0;
5953         adev->pm.dpm.dyn_state.valid_sclk_values.values = NULL;
5954         adev->pm.dpm.dyn_state.valid_mclk_values.count = 0;
5955         adev->pm.dpm.dyn_state.valid_mclk_values.values = NULL;
5956
5957         if (adev->asic_type == CHIP_HAWAII) {
5958                 pi->thermal_temp_setting.temperature_low = 94500;
5959                 pi->thermal_temp_setting.temperature_high = 95000;
5960                 pi->thermal_temp_setting.temperature_shutdown = 104000;
5961         } else {
5962                 pi->thermal_temp_setting.temperature_low = 99500;
5963                 pi->thermal_temp_setting.temperature_high = 100000;
5964                 pi->thermal_temp_setting.temperature_shutdown = 104000;
5965         }
5966
5967         pi->uvd_enabled = false;
5968
5969         dpm_table = &pi->smc_state_table;
5970
5971         gpio = amdgpu_atombios_lookup_gpio(adev, VDDC_VRHOT_GPIO_PINID);
5972         if (gpio.valid) {
5973                 dpm_table->VRHotGpio = gpio.shift;
5974                 adev->pm.dpm.platform_caps |= ATOM_PP_PLATFORM_CAP_REGULATOR_HOT;
5975         } else {
5976                 dpm_table->VRHotGpio = CISLANDS_UNUSED_GPIO_PIN;
5977                 adev->pm.dpm.platform_caps &= ~ATOM_PP_PLATFORM_CAP_REGULATOR_HOT;
5978         }
5979
5980         gpio = amdgpu_atombios_lookup_gpio(adev, PP_AC_DC_SWITCH_GPIO_PINID);
5981         if (gpio.valid) {
5982                 dpm_table->AcDcGpio = gpio.shift;
5983                 adev->pm.dpm.platform_caps |= ATOM_PP_PLATFORM_CAP_HARDWAREDC;
5984         } else {
5985                 dpm_table->AcDcGpio = CISLANDS_UNUSED_GPIO_PIN;
5986                 adev->pm.dpm.platform_caps &= ~ATOM_PP_PLATFORM_CAP_HARDWAREDC;
5987         }
5988
5989         gpio = amdgpu_atombios_lookup_gpio(adev, VDDC_PCC_GPIO_PINID);
5990         if (gpio.valid) {
5991                 u32 tmp = RREG32_SMC(ixCNB_PWRMGT_CNTL);
5992
5993                 switch (gpio.shift) {
5994                 case 0:
5995                         tmp &= ~CNB_PWRMGT_CNTL__GNB_SLOW_MODE_MASK;
5996                         tmp |= 1 << CNB_PWRMGT_CNTL__GNB_SLOW_MODE__SHIFT;
5997                         break;
5998                 case 1:
5999                         tmp &= ~CNB_PWRMGT_CNTL__GNB_SLOW_MODE_MASK;
6000                         tmp |= 2 << CNB_PWRMGT_CNTL__GNB_SLOW_MODE__SHIFT;
6001                         break;
6002                 case 2:
6003                         tmp |= CNB_PWRMGT_CNTL__GNB_SLOW_MASK;
6004                         break;
6005                 case 3:
6006                         tmp |= CNB_PWRMGT_CNTL__FORCE_NB_PS1_MASK;
6007                         break;
6008                 case 4:
6009                         tmp |= CNB_PWRMGT_CNTL__DPM_ENABLED_MASK;
6010                         break;
6011                 default:
6012                         DRM_ERROR("Invalid PCC GPIO: %u!\n", gpio.shift);
6013                         break;
6014                 }
6015                 WREG32_SMC(ixCNB_PWRMGT_CNTL, tmp);
6016         }
6017
6018         pi->voltage_control = CISLANDS_VOLTAGE_CONTROL_NONE;
6019         pi->vddci_control = CISLANDS_VOLTAGE_CONTROL_NONE;
6020         pi->mvdd_control = CISLANDS_VOLTAGE_CONTROL_NONE;
6021         if (amdgpu_atombios_is_voltage_gpio(adev, VOLTAGE_TYPE_VDDC, VOLTAGE_OBJ_GPIO_LUT))
6022                 pi->voltage_control = CISLANDS_VOLTAGE_CONTROL_BY_GPIO;
6023         else if (amdgpu_atombios_is_voltage_gpio(adev, VOLTAGE_TYPE_VDDC, VOLTAGE_OBJ_SVID2))
6024                 pi->voltage_control = CISLANDS_VOLTAGE_CONTROL_BY_SVID2;
6025
6026         if (adev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_VDDCI_CONTROL) {
6027                 if (amdgpu_atombios_is_voltage_gpio(adev, VOLTAGE_TYPE_VDDCI, VOLTAGE_OBJ_GPIO_LUT))
6028                         pi->vddci_control = CISLANDS_VOLTAGE_CONTROL_BY_GPIO;
6029                 else if (amdgpu_atombios_is_voltage_gpio(adev, VOLTAGE_TYPE_VDDCI, VOLTAGE_OBJ_SVID2))
6030                         pi->vddci_control = CISLANDS_VOLTAGE_CONTROL_BY_SVID2;
6031                 else
6032                         adev->pm.dpm.platform_caps &= ~ATOM_PP_PLATFORM_CAP_VDDCI_CONTROL;
6033         }
6034
6035         if (adev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_MVDDCONTROL) {
6036                 if (amdgpu_atombios_is_voltage_gpio(adev, VOLTAGE_TYPE_MVDDC, VOLTAGE_OBJ_GPIO_LUT))
6037                         pi->mvdd_control = CISLANDS_VOLTAGE_CONTROL_BY_GPIO;
6038                 else if (amdgpu_atombios_is_voltage_gpio(adev, VOLTAGE_TYPE_MVDDC, VOLTAGE_OBJ_SVID2))
6039                         pi->mvdd_control = CISLANDS_VOLTAGE_CONTROL_BY_SVID2;
6040                 else
6041                         adev->pm.dpm.platform_caps &= ~ATOM_PP_PLATFORM_CAP_MVDDCONTROL;
6042         }
6043
6044         pi->vddc_phase_shed_control = true;
6045
6046 #if defined(CONFIG_ACPI)
6047         pi->pcie_performance_request =
6048                 amdgpu_acpi_is_pcie_performance_request_supported(adev);
6049 #else
6050         pi->pcie_performance_request = false;
6051 #endif
6052
6053         if (amdgpu_atom_parse_data_header(adev->mode_info.atom_context, index, &size,
6054                                    &frev, &crev, &data_offset)) {
6055                 pi->caps_sclk_ss_support = true;
6056                 pi->caps_mclk_ss_support = true;
6057                 pi->dynamic_ss = true;
6058         } else {
6059                 pi->caps_sclk_ss_support = false;
6060                 pi->caps_mclk_ss_support = false;
6061                 pi->dynamic_ss = true;
6062         }
6063
6064         if (adev->pm.int_thermal_type != THERMAL_TYPE_NONE)
6065                 pi->thermal_protection = true;
6066         else
6067                 pi->thermal_protection = false;
6068
6069         pi->caps_dynamic_ac_timing = true;
6070
6071         pi->uvd_power_gated = false;
6072
6073         /* make sure dc limits are valid */
6074         if ((adev->pm.dpm.dyn_state.max_clock_voltage_on_dc.sclk == 0) ||
6075             (adev->pm.dpm.dyn_state.max_clock_voltage_on_dc.mclk == 0))
6076                 adev->pm.dpm.dyn_state.max_clock_voltage_on_dc =
6077                         adev->pm.dpm.dyn_state.max_clock_voltage_on_ac;
6078
6079         pi->fan_ctrl_is_in_default_mode = true;
6080
6081         return 0;
6082 }
6083
6084 static void
6085 ci_dpm_debugfs_print_current_performance_level(struct amdgpu_device *adev,
6086                                                struct seq_file *m)
6087 {
6088         struct ci_power_info *pi = ci_get_pi(adev);
6089         struct amdgpu_ps *rps = &pi->current_rps;
6090         u32 sclk = ci_get_average_sclk_freq(adev);
6091         u32 mclk = ci_get_average_mclk_freq(adev);
6092
6093         seq_printf(m, "uvd %sabled\n", pi->uvd_enabled ? "en" : "dis");
6094         seq_printf(m, "vce %sabled\n", rps->vce_active ? "en" : "dis");
6095         seq_printf(m, "power level avg    sclk: %u mclk: %u\n",
6096                    sclk, mclk);
6097 }
6098
6099 static void ci_dpm_print_power_state(struct amdgpu_device *adev,
6100                                      struct amdgpu_ps *rps)
6101 {
6102         struct ci_ps *ps = ci_get_ps(rps);
6103         struct ci_pl *pl;
6104         int i;
6105
6106         amdgpu_dpm_print_class_info(rps->class, rps->class2);
6107         amdgpu_dpm_print_cap_info(rps->caps);
6108         printk("\tuvd    vclk: %d dclk: %d\n", rps->vclk, rps->dclk);
6109         for (i = 0; i < ps->performance_level_count; i++) {
6110                 pl = &ps->performance_levels[i];
6111                 printk("\t\tpower level %d    sclk: %u mclk: %u pcie gen: %u pcie lanes: %u\n",
6112                        i, pl->sclk, pl->mclk, pl->pcie_gen + 1, pl->pcie_lane);
6113         }
6114         amdgpu_dpm_print_ps_status(adev, rps);
6115 }
6116
6117 static u32 ci_dpm_get_sclk(struct amdgpu_device *adev, bool low)
6118 {
6119         struct ci_power_info *pi = ci_get_pi(adev);
6120         struct ci_ps *requested_state = ci_get_ps(&pi->requested_rps);
6121
6122         if (low)
6123                 return requested_state->performance_levels[0].sclk;
6124         else
6125                 return requested_state->performance_levels[requested_state->performance_level_count - 1].sclk;
6126 }
6127
6128 static u32 ci_dpm_get_mclk(struct amdgpu_device *adev, bool low)
6129 {
6130         struct ci_power_info *pi = ci_get_pi(adev);
6131         struct ci_ps *requested_state = ci_get_ps(&pi->requested_rps);
6132
6133         if (low)
6134                 return requested_state->performance_levels[0].mclk;
6135         else
6136                 return requested_state->performance_levels[requested_state->performance_level_count - 1].mclk;
6137 }
6138
6139 /* get temperature in millidegrees */
6140 static int ci_dpm_get_temp(struct amdgpu_device *adev)
6141 {
6142         u32 temp;
6143         int actual_temp = 0;
6144
6145         temp = (RREG32_SMC(ixCG_MULT_THERMAL_STATUS) & CG_MULT_THERMAL_STATUS__CTF_TEMP_MASK) >>
6146                 CG_MULT_THERMAL_STATUS__CTF_TEMP__SHIFT;
6147
6148         if (temp & 0x200)
6149                 actual_temp = 255;
6150         else
6151                 actual_temp = temp & 0x1ff;
6152
6153         actual_temp = actual_temp * 1000;
6154
6155         return actual_temp;
6156 }
6157
6158 static int ci_set_temperature_range(struct amdgpu_device *adev)
6159 {
6160         int ret;
6161
6162         ret = ci_thermal_enable_alert(adev, false);
6163         if (ret)
6164                 return ret;
6165         ret = ci_thermal_set_temperature_range(adev, CISLANDS_TEMP_RANGE_MIN,
6166                                                CISLANDS_TEMP_RANGE_MAX);
6167         if (ret)
6168                 return ret;
6169         ret = ci_thermal_enable_alert(adev, true);
6170         if (ret)
6171                 return ret;
6172         return ret;
6173 }
6174
6175 static int ci_dpm_early_init(void *handle)
6176 {
6177         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
6178
6179         ci_dpm_set_dpm_funcs(adev);
6180         ci_dpm_set_irq_funcs(adev);
6181
6182         return 0;
6183 }
6184
6185 static int ci_dpm_late_init(void *handle)
6186 {
6187         int ret;
6188         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
6189
6190         if (!amdgpu_dpm)
6191                 return 0;
6192
6193         /* init the sysfs and debugfs files late */
6194         ret = amdgpu_pm_sysfs_init(adev);
6195         if (ret)
6196                 return ret;
6197
6198         ret = ci_set_temperature_range(adev);
6199         if (ret)
6200                 return ret;
6201
6202         ci_dpm_powergate_uvd(adev, true);
6203
6204         return 0;
6205 }
6206
6207 static int ci_dpm_sw_init(void *handle)
6208 {
6209         int ret;
6210         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
6211
6212         ret = amdgpu_irq_add_id(adev, 230, &adev->pm.dpm.thermal.irq);
6213         if (ret)
6214                 return ret;
6215
6216         ret = amdgpu_irq_add_id(adev, 231, &adev->pm.dpm.thermal.irq);
6217         if (ret)
6218                 return ret;
6219
6220         /* default to balanced state */
6221         adev->pm.dpm.state = POWER_STATE_TYPE_BALANCED;
6222         adev->pm.dpm.user_state = POWER_STATE_TYPE_BALANCED;
6223         adev->pm.dpm.forced_level = AMDGPU_DPM_FORCED_LEVEL_AUTO;
6224         adev->pm.default_sclk = adev->clock.default_sclk;
6225         adev->pm.default_mclk = adev->clock.default_mclk;
6226         adev->pm.current_sclk = adev->clock.default_sclk;
6227         adev->pm.current_mclk = adev->clock.default_mclk;
6228         adev->pm.int_thermal_type = THERMAL_TYPE_NONE;
6229
6230         if (amdgpu_dpm == 0)
6231                 return 0;
6232
6233         ret = ci_dpm_init_microcode(adev);
6234         if (ret)
6235                 return ret;
6236
6237         INIT_WORK(&adev->pm.dpm.thermal.work, amdgpu_dpm_thermal_work_handler);
6238         mutex_lock(&adev->pm.mutex);
6239         ret = ci_dpm_init(adev);
6240         if (ret)
6241                 goto dpm_failed;
6242         adev->pm.dpm.current_ps = adev->pm.dpm.requested_ps = adev->pm.dpm.boot_ps;
6243         if (amdgpu_dpm == 1)
6244                 amdgpu_pm_print_power_states(adev);
6245         mutex_unlock(&adev->pm.mutex);
6246         DRM_INFO("amdgpu: dpm initialized\n");
6247
6248         return 0;
6249
6250 dpm_failed:
6251         ci_dpm_fini(adev);
6252         mutex_unlock(&adev->pm.mutex);
6253         DRM_ERROR("amdgpu: dpm initialization failed\n");
6254         return ret;
6255 }
6256
6257 static int ci_dpm_sw_fini(void *handle)
6258 {
6259         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
6260
6261         mutex_lock(&adev->pm.mutex);
6262         amdgpu_pm_sysfs_fini(adev);
6263         ci_dpm_fini(adev);
6264         mutex_unlock(&adev->pm.mutex);
6265
6266         return 0;
6267 }
6268
6269 static int ci_dpm_hw_init(void *handle)
6270 {
6271         int ret;
6272
6273         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
6274
6275         if (!amdgpu_dpm)
6276                 return 0;
6277
6278         mutex_lock(&adev->pm.mutex);
6279         ci_dpm_setup_asic(adev);
6280         ret = ci_dpm_enable(adev);
6281         if (ret)
6282                 adev->pm.dpm_enabled = false;
6283         else
6284                 adev->pm.dpm_enabled = true;
6285         mutex_unlock(&adev->pm.mutex);
6286
6287         return ret;
6288 }
6289
6290 static int ci_dpm_hw_fini(void *handle)
6291 {
6292         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
6293
6294         if (adev->pm.dpm_enabled) {
6295                 mutex_lock(&adev->pm.mutex);
6296                 ci_dpm_disable(adev);
6297                 mutex_unlock(&adev->pm.mutex);
6298         }
6299
6300         return 0;
6301 }
6302
6303 static int ci_dpm_suspend(void *handle)
6304 {
6305         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
6306
6307         if (adev->pm.dpm_enabled) {
6308                 mutex_lock(&adev->pm.mutex);
6309                 /* disable dpm */
6310                 ci_dpm_disable(adev);
6311                 /* reset the power state */
6312                 adev->pm.dpm.current_ps = adev->pm.dpm.requested_ps = adev->pm.dpm.boot_ps;
6313                 mutex_unlock(&adev->pm.mutex);
6314         }
6315         return 0;
6316 }
6317
6318 static int ci_dpm_resume(void *handle)
6319 {
6320         int ret;
6321         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
6322
6323         if (adev->pm.dpm_enabled) {
6324                 /* asic init will reset to the boot state */
6325                 mutex_lock(&adev->pm.mutex);
6326                 ci_dpm_setup_asic(adev);
6327                 ret = ci_dpm_enable(adev);
6328                 if (ret)
6329                         adev->pm.dpm_enabled = false;
6330                 else
6331                         adev->pm.dpm_enabled = true;
6332                 mutex_unlock(&adev->pm.mutex);
6333                 if (adev->pm.dpm_enabled)
6334                         amdgpu_pm_compute_clocks(adev);
6335         }
6336         return 0;
6337 }
6338
6339 static bool ci_dpm_is_idle(void *handle)
6340 {
6341         /* XXX */
6342         return true;
6343 }
6344
6345 static int ci_dpm_wait_for_idle(void *handle)
6346 {
6347         /* XXX */
6348         return 0;
6349 }
6350
6351 static void ci_dpm_print_status(void *handle)
6352 {
6353         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
6354
6355         dev_info(adev->dev, "CIK DPM registers\n");
6356         dev_info(adev->dev, "  BIOS_SCRATCH_4=0x%08X\n",
6357                  RREG32(mmBIOS_SCRATCH_4));
6358         dev_info(adev->dev, "  MC_ARB_DRAM_TIMING=0x%08X\n",
6359                  RREG32(mmMC_ARB_DRAM_TIMING));
6360         dev_info(adev->dev, "  MC_ARB_DRAM_TIMING2=0x%08X\n",
6361                  RREG32(mmMC_ARB_DRAM_TIMING2));
6362         dev_info(adev->dev, "  MC_ARB_BURST_TIME=0x%08X\n",
6363                  RREG32(mmMC_ARB_BURST_TIME));
6364         dev_info(adev->dev, "  MC_ARB_DRAM_TIMING_1=0x%08X\n",
6365                  RREG32(mmMC_ARB_DRAM_TIMING_1));
6366         dev_info(adev->dev, "  MC_ARB_DRAM_TIMING2_1=0x%08X\n",
6367                  RREG32(mmMC_ARB_DRAM_TIMING2_1));
6368         dev_info(adev->dev, "  MC_CG_CONFIG=0x%08X\n",
6369                  RREG32(mmMC_CG_CONFIG));
6370         dev_info(adev->dev, "  MC_ARB_CG=0x%08X\n",
6371                  RREG32(mmMC_ARB_CG));
6372         dev_info(adev->dev, "  DIDT_SQ_CTRL0=0x%08X\n",
6373                  RREG32_DIDT(ixDIDT_SQ_CTRL0));
6374         dev_info(adev->dev, "  DIDT_DB_CTRL0=0x%08X\n",
6375                  RREG32_DIDT(ixDIDT_DB_CTRL0));
6376         dev_info(adev->dev, "  DIDT_TD_CTRL0=0x%08X\n",
6377                  RREG32_DIDT(ixDIDT_TD_CTRL0));
6378         dev_info(adev->dev, "  DIDT_TCP_CTRL0=0x%08X\n",
6379                  RREG32_DIDT(ixDIDT_TCP_CTRL0));
6380         dev_info(adev->dev, "  CG_THERMAL_INT=0x%08X\n",
6381                  RREG32_SMC(ixCG_THERMAL_INT));
6382         dev_info(adev->dev, "  CG_THERMAL_CTRL=0x%08X\n",
6383                  RREG32_SMC(ixCG_THERMAL_CTRL));
6384         dev_info(adev->dev, "  GENERAL_PWRMGT=0x%08X\n",
6385                  RREG32_SMC(ixGENERAL_PWRMGT));
6386         dev_info(adev->dev, "  MC_SEQ_CNTL_3=0x%08X\n",
6387                  RREG32(mmMC_SEQ_CNTL_3));
6388         dev_info(adev->dev, "  LCAC_MC0_CNTL=0x%08X\n",
6389                  RREG32_SMC(ixLCAC_MC0_CNTL));
6390         dev_info(adev->dev, "  LCAC_MC1_CNTL=0x%08X\n",
6391                  RREG32_SMC(ixLCAC_MC1_CNTL));
6392         dev_info(adev->dev, "  LCAC_CPL_CNTL=0x%08X\n",
6393                  RREG32_SMC(ixLCAC_CPL_CNTL));
6394         dev_info(adev->dev, "  SCLK_PWRMGT_CNTL=0x%08X\n",
6395                  RREG32_SMC(ixSCLK_PWRMGT_CNTL));
6396         dev_info(adev->dev, "  BIF_LNCNT_RESET=0x%08X\n",
6397                  RREG32(mmBIF_LNCNT_RESET));
6398         dev_info(adev->dev, "  FIRMWARE_FLAGS=0x%08X\n",
6399                  RREG32_SMC(ixFIRMWARE_FLAGS));
6400         dev_info(adev->dev, "  CG_SPLL_FUNC_CNTL=0x%08X\n",
6401                  RREG32_SMC(ixCG_SPLL_FUNC_CNTL));
6402         dev_info(adev->dev, "  CG_SPLL_FUNC_CNTL_2=0x%08X\n",
6403                  RREG32_SMC(ixCG_SPLL_FUNC_CNTL_2));
6404         dev_info(adev->dev, "  CG_SPLL_FUNC_CNTL_3=0x%08X\n",
6405                  RREG32_SMC(ixCG_SPLL_FUNC_CNTL_3));
6406         dev_info(adev->dev, "  CG_SPLL_FUNC_CNTL_4=0x%08X\n",
6407                  RREG32_SMC(ixCG_SPLL_FUNC_CNTL_4));
6408         dev_info(adev->dev, "  CG_SPLL_SPREAD_SPECTRUM=0x%08X\n",
6409                  RREG32_SMC(ixCG_SPLL_SPREAD_SPECTRUM));
6410         dev_info(adev->dev, "  CG_SPLL_SPREAD_SPECTRUM_2=0x%08X\n",
6411                  RREG32_SMC(ixCG_SPLL_SPREAD_SPECTRUM_2));
6412         dev_info(adev->dev, "  DLL_CNTL=0x%08X\n",
6413                  RREG32(mmDLL_CNTL));
6414         dev_info(adev->dev, "  MCLK_PWRMGT_CNTL=0x%08X\n",
6415                  RREG32(mmMCLK_PWRMGT_CNTL));
6416         dev_info(adev->dev, "  MPLL_AD_FUNC_CNTL=0x%08X\n",
6417                  RREG32(mmMPLL_AD_FUNC_CNTL));
6418         dev_info(adev->dev, "  MPLL_DQ_FUNC_CNTL=0x%08X\n",
6419                  RREG32(mmMPLL_DQ_FUNC_CNTL));
6420         dev_info(adev->dev, "  MPLL_FUNC_CNTL=0x%08X\n",
6421                  RREG32(mmMPLL_FUNC_CNTL));
6422         dev_info(adev->dev, "  MPLL_FUNC_CNTL_1=0x%08X\n",
6423                  RREG32(mmMPLL_FUNC_CNTL_1));
6424         dev_info(adev->dev, "  MPLL_FUNC_CNTL_2=0x%08X\n",
6425                  RREG32(mmMPLL_FUNC_CNTL_2));
6426         dev_info(adev->dev, "  MPLL_SS1=0x%08X\n",
6427                  RREG32(mmMPLL_SS1));
6428         dev_info(adev->dev, "  MPLL_SS2=0x%08X\n",
6429                  RREG32(mmMPLL_SS2));
6430         dev_info(adev->dev, "  CG_DISPLAY_GAP_CNTL=0x%08X\n",
6431                  RREG32_SMC(ixCG_DISPLAY_GAP_CNTL));
6432         dev_info(adev->dev, "  CG_DISPLAY_GAP_CNTL2=0x%08X\n",
6433                  RREG32_SMC(ixCG_DISPLAY_GAP_CNTL2));
6434         dev_info(adev->dev, "  CG_STATIC_SCREEN_PARAMETER=0x%08X\n",
6435                  RREG32_SMC(ixCG_STATIC_SCREEN_PARAMETER));
6436         dev_info(adev->dev, "  CG_FREQ_TRAN_VOTING_0=0x%08X\n",
6437                  RREG32_SMC(ixCG_FREQ_TRAN_VOTING_0));
6438         dev_info(adev->dev, "  CG_FREQ_TRAN_VOTING_1=0x%08X\n",
6439                  RREG32_SMC(ixCG_FREQ_TRAN_VOTING_1));
6440         dev_info(adev->dev, "  CG_FREQ_TRAN_VOTING_2=0x%08X\n",
6441                  RREG32_SMC(ixCG_FREQ_TRAN_VOTING_2));
6442         dev_info(adev->dev, "  CG_FREQ_TRAN_VOTING_3=0x%08X\n",
6443                  RREG32_SMC(ixCG_FREQ_TRAN_VOTING_3));
6444         dev_info(adev->dev, "  CG_FREQ_TRAN_VOTING_4=0x%08X\n",
6445                  RREG32_SMC(ixCG_FREQ_TRAN_VOTING_4));
6446         dev_info(adev->dev, "  CG_FREQ_TRAN_VOTING_5=0x%08X\n",
6447                  RREG32_SMC(ixCG_FREQ_TRAN_VOTING_5));
6448         dev_info(adev->dev, "  CG_FREQ_TRAN_VOTING_6=0x%08X\n",
6449                  RREG32_SMC(ixCG_FREQ_TRAN_VOTING_6));
6450         dev_info(adev->dev, "  CG_FREQ_TRAN_VOTING_7=0x%08X\n",
6451                  RREG32_SMC(ixCG_FREQ_TRAN_VOTING_7));
6452         dev_info(adev->dev, "  RCU_UC_EVENTS=0x%08X\n",
6453                  RREG32_SMC(ixRCU_UC_EVENTS));
6454         dev_info(adev->dev, "  DPM_TABLE_475=0x%08X\n",
6455                  RREG32_SMC(ixDPM_TABLE_475));
6456         dev_info(adev->dev, "  MC_SEQ_RAS_TIMING_LP=0x%08X\n",
6457                  RREG32(mmMC_SEQ_RAS_TIMING_LP));
6458         dev_info(adev->dev, "  MC_SEQ_RAS_TIMING=0x%08X\n",
6459                  RREG32(mmMC_SEQ_RAS_TIMING));
6460         dev_info(adev->dev, "  MC_SEQ_CAS_TIMING_LP=0x%08X\n",
6461                  RREG32(mmMC_SEQ_CAS_TIMING_LP));
6462         dev_info(adev->dev, "  MC_SEQ_CAS_TIMING=0x%08X\n",
6463                  RREG32(mmMC_SEQ_CAS_TIMING));
6464         dev_info(adev->dev, "  MC_SEQ_DLL_STBY_LP=0x%08X\n",
6465                  RREG32(mmMC_SEQ_DLL_STBY_LP));
6466         dev_info(adev->dev, "  MC_SEQ_DLL_STBY=0x%08X\n",
6467                  RREG32(mmMC_SEQ_DLL_STBY));
6468         dev_info(adev->dev, "  MC_SEQ_G5PDX_CMD0_LP=0x%08X\n",
6469                  RREG32(mmMC_SEQ_G5PDX_CMD0_LP));
6470         dev_info(adev->dev, "  MC_SEQ_G5PDX_CMD0=0x%08X\n",
6471                  RREG32(mmMC_SEQ_G5PDX_CMD0));
6472         dev_info(adev->dev, "  MC_SEQ_G5PDX_CMD1_LP=0x%08X\n",
6473                  RREG32(mmMC_SEQ_G5PDX_CMD1_LP));
6474         dev_info(adev->dev, "  MC_SEQ_G5PDX_CMD1=0x%08X\n",
6475                  RREG32(mmMC_SEQ_G5PDX_CMD1));
6476         dev_info(adev->dev, "  MC_SEQ_G5PDX_CTRL_LP=0x%08X\n",
6477                  RREG32(mmMC_SEQ_G5PDX_CTRL_LP));
6478         dev_info(adev->dev, "  MC_SEQ_G5PDX_CTRL=0x%08X\n",
6479                  RREG32(mmMC_SEQ_G5PDX_CTRL));
6480         dev_info(adev->dev, "  MC_SEQ_PMG_DVS_CMD_LP=0x%08X\n",
6481                  RREG32(mmMC_SEQ_PMG_DVS_CMD_LP));
6482         dev_info(adev->dev, "  MC_SEQ_PMG_DVS_CMD=0x%08X\n",
6483                  RREG32(mmMC_SEQ_PMG_DVS_CMD));
6484         dev_info(adev->dev, "  MC_SEQ_PMG_DVS_CTL_LP=0x%08X\n",
6485                  RREG32(mmMC_SEQ_PMG_DVS_CTL_LP));
6486         dev_info(adev->dev, "  MC_SEQ_PMG_DVS_CTL=0x%08X\n",
6487                  RREG32(mmMC_SEQ_PMG_DVS_CTL));
6488         dev_info(adev->dev, "  MC_SEQ_MISC_TIMING_LP=0x%08X\n",
6489                  RREG32(mmMC_SEQ_MISC_TIMING_LP));
6490         dev_info(adev->dev, "  MC_SEQ_MISC_TIMING=0x%08X\n",
6491                  RREG32(mmMC_SEQ_MISC_TIMING));
6492         dev_info(adev->dev, "  MC_SEQ_MISC_TIMING2_LP=0x%08X\n",
6493                  RREG32(mmMC_SEQ_MISC_TIMING2_LP));
6494         dev_info(adev->dev, "  MC_SEQ_MISC_TIMING2=0x%08X\n",
6495                  RREG32(mmMC_SEQ_MISC_TIMING2));
6496         dev_info(adev->dev, "  MC_SEQ_PMG_CMD_EMRS_LP=0x%08X\n",
6497                  RREG32(mmMC_SEQ_PMG_CMD_EMRS_LP));
6498         dev_info(adev->dev, "  MC_PMG_CMD_EMRS=0x%08X\n",
6499                  RREG32(mmMC_PMG_CMD_EMRS));
6500         dev_info(adev->dev, "  MC_SEQ_PMG_CMD_MRS_LP=0x%08X\n",
6501                  RREG32(mmMC_SEQ_PMG_CMD_MRS_LP));
6502         dev_info(adev->dev, "  MC_PMG_CMD_MRS=0x%08X\n",
6503                  RREG32(mmMC_PMG_CMD_MRS));
6504         dev_info(adev->dev, "  MC_SEQ_PMG_CMD_MRS1_LP=0x%08X\n",
6505                  RREG32(mmMC_SEQ_PMG_CMD_MRS1_LP));
6506         dev_info(adev->dev, "  MC_PMG_CMD_MRS1=0x%08X\n",
6507                  RREG32(mmMC_PMG_CMD_MRS1));
6508         dev_info(adev->dev, "  MC_SEQ_WR_CTL_D0_LP=0x%08X\n",
6509                  RREG32(mmMC_SEQ_WR_CTL_D0_LP));
6510         dev_info(adev->dev, "  MC_SEQ_WR_CTL_D0=0x%08X\n",
6511                  RREG32(mmMC_SEQ_WR_CTL_D0));
6512         dev_info(adev->dev, "  MC_SEQ_WR_CTL_D1_LP=0x%08X\n",
6513                  RREG32(mmMC_SEQ_WR_CTL_D1_LP));
6514         dev_info(adev->dev, "  MC_SEQ_WR_CTL_D1=0x%08X\n",
6515                  RREG32(mmMC_SEQ_WR_CTL_D1));
6516         dev_info(adev->dev, "  MC_SEQ_RD_CTL_D0_LP=0x%08X\n",
6517                  RREG32(mmMC_SEQ_RD_CTL_D0_LP));
6518         dev_info(adev->dev, "  MC_SEQ_RD_CTL_D0=0x%08X\n",
6519                  RREG32(mmMC_SEQ_RD_CTL_D0));
6520         dev_info(adev->dev, "  MC_SEQ_RD_CTL_D1_LP=0x%08X\n",
6521                  RREG32(mmMC_SEQ_RD_CTL_D1_LP));
6522         dev_info(adev->dev, "  MC_SEQ_RD_CTL_D1=0x%08X\n",
6523                  RREG32(mmMC_SEQ_RD_CTL_D1));
6524         dev_info(adev->dev, "  MC_SEQ_PMG_TIMING_LP=0x%08X\n",
6525                  RREG32(mmMC_SEQ_PMG_TIMING_LP));
6526         dev_info(adev->dev, "  MC_SEQ_PMG_TIMING=0x%08X\n",
6527                  RREG32(mmMC_SEQ_PMG_TIMING));
6528         dev_info(adev->dev, "  MC_SEQ_PMG_CMD_MRS2_LP=0x%08X\n",
6529                  RREG32(mmMC_SEQ_PMG_CMD_MRS2_LP));
6530         dev_info(adev->dev, "  MC_PMG_CMD_MRS2=0x%08X\n",
6531                  RREG32(mmMC_PMG_CMD_MRS2));
6532         dev_info(adev->dev, "  MC_SEQ_WR_CTL_2_LP=0x%08X\n",
6533                  RREG32(mmMC_SEQ_WR_CTL_2_LP));
6534         dev_info(adev->dev, "  MC_SEQ_WR_CTL_2=0x%08X\n",
6535                  RREG32(mmMC_SEQ_WR_CTL_2));
6536         dev_info(adev->dev, "  PCIE_LC_SPEED_CNTL=0x%08X\n",
6537                  RREG32_PCIE(ixPCIE_LC_SPEED_CNTL));
6538         dev_info(adev->dev, "  PCIE_LC_LINK_WIDTH_CNTL=0x%08X\n",
6539                  RREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL));
6540         dev_info(adev->dev, "  SMC_IND_INDEX_0=0x%08X\n",
6541                  RREG32(mmSMC_IND_INDEX_0));
6542         dev_info(adev->dev, "  SMC_IND_DATA_0=0x%08X\n",
6543                  RREG32(mmSMC_IND_DATA_0));
6544         dev_info(adev->dev, "  SMC_IND_ACCESS_CNTL=0x%08X\n",
6545                  RREG32(mmSMC_IND_ACCESS_CNTL));
6546         dev_info(adev->dev, "  SMC_RESP_0=0x%08X\n",
6547                  RREG32(mmSMC_RESP_0));
6548         dev_info(adev->dev, "  SMC_MESSAGE_0=0x%08X\n",
6549                  RREG32(mmSMC_MESSAGE_0));
6550         dev_info(adev->dev, "  SMC_SYSCON_RESET_CNTL=0x%08X\n",
6551                  RREG32_SMC(ixSMC_SYSCON_RESET_CNTL));
6552         dev_info(adev->dev, "  SMC_SYSCON_CLOCK_CNTL_0=0x%08X\n",
6553                  RREG32_SMC(ixSMC_SYSCON_CLOCK_CNTL_0));
6554         dev_info(adev->dev, "  SMC_SYSCON_MISC_CNTL=0x%08X\n",
6555                  RREG32_SMC(ixSMC_SYSCON_MISC_CNTL));
6556         dev_info(adev->dev, "  SMC_PC_C=0x%08X\n",
6557                  RREG32_SMC(ixSMC_PC_C));
6558 }
6559
6560 static int ci_dpm_soft_reset(void *handle)
6561 {
6562         return 0;
6563 }
6564
6565 static int ci_dpm_set_interrupt_state(struct amdgpu_device *adev,
6566                                       struct amdgpu_irq_src *source,
6567                                       unsigned type,
6568                                       enum amdgpu_interrupt_state state)
6569 {
6570         u32 cg_thermal_int;
6571
6572         switch (type) {
6573         case AMDGPU_THERMAL_IRQ_LOW_TO_HIGH:
6574                 switch (state) {
6575                 case AMDGPU_IRQ_STATE_DISABLE:
6576                         cg_thermal_int = RREG32_SMC(ixCG_THERMAL_INT);
6577                         cg_thermal_int |= CG_THERMAL_INT_CTRL__THERM_INTH_MASK_MASK;
6578                         WREG32_SMC(ixCG_THERMAL_INT, cg_thermal_int);
6579                         break;
6580                 case AMDGPU_IRQ_STATE_ENABLE:
6581                         cg_thermal_int = RREG32_SMC(ixCG_THERMAL_INT);
6582                         cg_thermal_int &= ~CG_THERMAL_INT_CTRL__THERM_INTH_MASK_MASK;
6583                         WREG32_SMC(ixCG_THERMAL_INT, cg_thermal_int);
6584                         break;
6585                 default:
6586                         break;
6587                 }
6588                 break;
6589
6590         case AMDGPU_THERMAL_IRQ_HIGH_TO_LOW:
6591                 switch (state) {
6592                 case AMDGPU_IRQ_STATE_DISABLE:
6593                         cg_thermal_int = RREG32_SMC(ixCG_THERMAL_INT);
6594                         cg_thermal_int |= CG_THERMAL_INT_CTRL__THERM_INTL_MASK_MASK;
6595                         WREG32_SMC(ixCG_THERMAL_INT, cg_thermal_int);
6596                         break;
6597                 case AMDGPU_IRQ_STATE_ENABLE:
6598                         cg_thermal_int = RREG32_SMC(ixCG_THERMAL_INT);
6599                         cg_thermal_int &= ~CG_THERMAL_INT_CTRL__THERM_INTL_MASK_MASK;
6600                         WREG32_SMC(ixCG_THERMAL_INT, cg_thermal_int);
6601                         break;
6602                 default:
6603                         break;
6604                 }
6605                 break;
6606
6607         default:
6608                 break;
6609         }
6610         return 0;
6611 }
6612
6613 static int ci_dpm_process_interrupt(struct amdgpu_device *adev,
6614                                     struct amdgpu_irq_src *source,
6615                                     struct amdgpu_iv_entry *entry)
6616 {
6617         bool queue_thermal = false;
6618
6619         if (entry == NULL)
6620                 return -EINVAL;
6621
6622         switch (entry->src_id) {
6623         case 230: /* thermal low to high */
6624                 DRM_DEBUG("IH: thermal low to high\n");
6625                 adev->pm.dpm.thermal.high_to_low = false;
6626                 queue_thermal = true;
6627                 break;
6628         case 231: /* thermal high to low */
6629                 DRM_DEBUG("IH: thermal high to low\n");
6630                 adev->pm.dpm.thermal.high_to_low = true;
6631                 queue_thermal = true;
6632                 break;
6633         default:
6634                 break;
6635         }
6636
6637         if (queue_thermal)
6638                 schedule_work(&adev->pm.dpm.thermal.work);
6639
6640         return 0;
6641 }
6642
6643 static int ci_dpm_set_clockgating_state(void *handle,
6644                                           enum amd_clockgating_state state)
6645 {
6646         return 0;
6647 }
6648
6649 static int ci_dpm_set_powergating_state(void *handle,
6650                                           enum amd_powergating_state state)
6651 {
6652         return 0;
6653 }
6654
6655 const struct amd_ip_funcs ci_dpm_ip_funcs = {
6656         .early_init = ci_dpm_early_init,
6657         .late_init = ci_dpm_late_init,
6658         .sw_init = ci_dpm_sw_init,
6659         .sw_fini = ci_dpm_sw_fini,
6660         .hw_init = ci_dpm_hw_init,
6661         .hw_fini = ci_dpm_hw_fini,
6662         .suspend = ci_dpm_suspend,
6663         .resume = ci_dpm_resume,
6664         .is_idle = ci_dpm_is_idle,
6665         .wait_for_idle = ci_dpm_wait_for_idle,
6666         .soft_reset = ci_dpm_soft_reset,
6667         .print_status = ci_dpm_print_status,
6668         .set_clockgating_state = ci_dpm_set_clockgating_state,
6669         .set_powergating_state = ci_dpm_set_powergating_state,
6670 };
6671
6672 static const struct amdgpu_dpm_funcs ci_dpm_funcs = {
6673         .get_temperature = &ci_dpm_get_temp,
6674         .pre_set_power_state = &ci_dpm_pre_set_power_state,
6675         .set_power_state = &ci_dpm_set_power_state,
6676         .post_set_power_state = &ci_dpm_post_set_power_state,
6677         .display_configuration_changed = &ci_dpm_display_configuration_changed,
6678         .get_sclk = &ci_dpm_get_sclk,
6679         .get_mclk = &ci_dpm_get_mclk,
6680         .print_power_state = &ci_dpm_print_power_state,
6681         .debugfs_print_current_performance_level = &ci_dpm_debugfs_print_current_performance_level,
6682         .force_performance_level = &ci_dpm_force_performance_level,
6683         .vblank_too_short = &ci_dpm_vblank_too_short,
6684         .powergate_uvd = &ci_dpm_powergate_uvd,
6685         .set_fan_control_mode = &ci_dpm_set_fan_control_mode,
6686         .get_fan_control_mode = &ci_dpm_get_fan_control_mode,
6687         .set_fan_speed_percent = &ci_dpm_set_fan_speed_percent,
6688         .get_fan_speed_percent = &ci_dpm_get_fan_speed_percent,
6689 };
6690
6691 static void ci_dpm_set_dpm_funcs(struct amdgpu_device *adev)
6692 {
6693         if (adev->pm.funcs == NULL)
6694                 adev->pm.funcs = &ci_dpm_funcs;
6695 }
6696
6697 static const struct amdgpu_irq_src_funcs ci_dpm_irq_funcs = {
6698         .set = ci_dpm_set_interrupt_state,
6699         .process = ci_dpm_process_interrupt,
6700 };
6701
6702 static void ci_dpm_set_irq_funcs(struct amdgpu_device *adev)
6703 {
6704         adev->pm.dpm.thermal.irq.num_types = AMDGPU_THERMAL_IRQ_LAST;
6705         adev->pm.dpm.thermal.irq.funcs = &ci_dpm_irq_funcs;
6706 }