drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c

   1 /*
   2  * Copyright 2014 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  */
  23
  24 #include <linux/firmware.h>
  25 #include <linux/module.h>
  26
  27 #include "amdgpu.h"
  28 #include "amdgpu_ih.h"
  29 #include "amdgpu_gfx.h"
  30 #include "cikd.h"
  31 #include "cik.h"
  32 #include "cik_structs.h"
  33 #include "atom.h"
  34 #include "amdgpu_ucode.h"
  35 #include "clearstate_ci.h"
  36
  37 #include "dce/dce_8_0_d.h"
  38 #include "dce/dce_8_0_sh_mask.h"
  39
  40 #include "bif/bif_4_1_d.h"
  41 #include "bif/bif_4_1_sh_mask.h"
  42
  43 #include "gca/gfx_7_0_d.h"
  44 #include "gca/gfx_7_2_enum.h"
  45 #include "gca/gfx_7_2_sh_mask.h"
  46
  47 #include "gmc/gmc_7_0_d.h"
  48 #include "gmc/gmc_7_0_sh_mask.h"
  49
  50 #include "oss/oss_2_0_d.h"
  51 #include "oss/oss_2_0_sh_mask.h"
  52
  53 #define NUM_SIMD_PER_CU 0x4 /* missing from the gfx_7 IP headers */
  54
  55 #define GFX7_NUM_GFX_RINGS     1
  56 #define GFX7_MEC_HPD_SIZE      2048
  57
  58 static void gfx_v7_0_set_ring_funcs(struct amdgpu_device *adev);
  59 static void gfx_v7_0_set_irq_funcs(struct amdgpu_device *adev);
  60 static void gfx_v7_0_set_gds_init(struct amdgpu_device *adev);
  61
  62 /*(DEBLOBBED)*/
  63
  64 static const struct amdgpu_gds_reg_offset amdgpu_gds_reg_offset[] =
  65 {
  66         {mmGDS_VMID0_BASE, mmGDS_VMID0_SIZE, mmGDS_GWS_VMID0, mmGDS_OA_VMID0},
  67         {mmGDS_VMID1_BASE, mmGDS_VMID1_SIZE, mmGDS_GWS_VMID1, mmGDS_OA_VMID1},
  68         {mmGDS_VMID2_BASE, mmGDS_VMID2_SIZE, mmGDS_GWS_VMID2, mmGDS_OA_VMID2},
  69         {mmGDS_VMID3_BASE, mmGDS_VMID3_SIZE, mmGDS_GWS_VMID3, mmGDS_OA_VMID3},
  70         {mmGDS_VMID4_BASE, mmGDS_VMID4_SIZE, mmGDS_GWS_VMID4, mmGDS_OA_VMID4},
  71         {mmGDS_VMID5_BASE, mmGDS_VMID5_SIZE, mmGDS_GWS_VMID5, mmGDS_OA_VMID5},
  72         {mmGDS_VMID6_BASE, mmGDS_VMID6_SIZE, mmGDS_GWS_VMID6, mmGDS_OA_VMID6},
  73         {mmGDS_VMID7_BASE, mmGDS_VMID7_SIZE, mmGDS_GWS_VMID7, mmGDS_OA_VMID7},
  74         {mmGDS_VMID8_BASE, mmGDS_VMID8_SIZE, mmGDS_GWS_VMID8, mmGDS_OA_VMID8},
  75         {mmGDS_VMID9_BASE, mmGDS_VMID9_SIZE, mmGDS_GWS_VMID9, mmGDS_OA_VMID9},
  76         {mmGDS_VMID10_BASE, mmGDS_VMID10_SIZE, mmGDS_GWS_VMID10, mmGDS_OA_VMID10},
  77         {mmGDS_VMID11_BASE, mmGDS_VMID11_SIZE, mmGDS_GWS_VMID11, mmGDS_OA_VMID11},
  78         {mmGDS_VMID12_BASE, mmGDS_VMID12_SIZE, mmGDS_GWS_VMID12, mmGDS_OA_VMID12},
  79         {mmGDS_VMID13_BASE, mmGDS_VMID13_SIZE, mmGDS_GWS_VMID13, mmGDS_OA_VMID13},
  80         {mmGDS_VMID14_BASE, mmGDS_VMID14_SIZE, mmGDS_GWS_VMID14, mmGDS_OA_VMID14},
  81         {mmGDS_VMID15_BASE, mmGDS_VMID15_SIZE, mmGDS_GWS_VMID15, mmGDS_OA_VMID15}
  82 };
  83
  84 static const u32 spectre_rlc_save_restore_register_list[] =
  85 {
  86         (0x0e00 << 16) | (0xc12c >> 2),
  87         0x00000000,
  88         (0x0e00 << 16) | (0xc140 >> 2),
  89         0x00000000,
  90         (0x0e00 << 16) | (0xc150 >> 2),
  91         0x00000000,
  92         (0x0e00 << 16) | (0xc15c >> 2),
  93         0x00000000,
  94         (0x0e00 << 16) | (0xc168 >> 2),
  95         0x00000000,
  96         (0x0e00 << 16) | (0xc170 >> 2),
  97         0x00000000,
  98         (0x0e00 << 16) | (0xc178 >> 2),
  99         0x00000000,
 100         (0x0e00 << 16) | (0xc204 >> 2),
 101         0x00000000,
 102         (0x0e00 << 16) | (0xc2b4 >> 2),
 103         0x00000000,
 104         (0x0e00 << 16) | (0xc2b8 >> 2),
 105         0x00000000,
 106         (0x0e00 << 16) | (0xc2bc >> 2),
 107         0x00000000,
 108         (0x0e00 << 16) | (0xc2c0 >> 2),
 109         0x00000000,
 110         (0x0e00 << 16) | (0x8228 >> 2),
 111         0x00000000,
 112         (0x0e00 << 16) | (0x829c >> 2),
 113         0x00000000,
 114         (0x0e00 << 16) | (0x869c >> 2),
 115         0x00000000,
 116         (0x0600 << 16) | (0x98f4 >> 2),
 117         0x00000000,
 118         (0x0e00 << 16) | (0x98f8 >> 2),
 119         0x00000000,
 120         (0x0e00 << 16) | (0x9900 >> 2),
 121         0x00000000,
 122         (0x0e00 << 16) | (0xc260 >> 2),
 123         0x00000000,
 124         (0x0e00 << 16) | (0x90e8 >> 2),
 125         0x00000000,
 126         (0x0e00 << 16) | (0x3c000 >> 2),
 127         0x00000000,
 128         (0x0e00 << 16) | (0x3c00c >> 2),
 129         0x00000000,
 130         (0x0e00 << 16) | (0x8c1c >> 2),
 131         0x00000000,
 132         (0x0e00 << 16) | (0x9700 >> 2),
 133         0x00000000,
 134         (0x0e00 << 16) | (0xcd20 >> 2),
 135         0x00000000,
 136         (0x4e00 << 16) | (0xcd20 >> 2),
 137         0x00000000,
 138         (0x5e00 << 16) | (0xcd20 >> 2),
 139         0x00000000,
 140         (0x6e00 << 16) | (0xcd20 >> 2),
 141         0x00000000,
 142         (0x7e00 << 16) | (0xcd20 >> 2),
 143         0x00000000,
 144         (0x8e00 << 16) | (0xcd20 >> 2),
 145         0x00000000,
 146         (0x9e00 << 16) | (0xcd20 >> 2),
 147         0x00000000,
 148         (0xae00 << 16) | (0xcd20 >> 2),
 149         0x00000000,
 150         (0xbe00 << 16) | (0xcd20 >> 2),
 151         0x00000000,
 152         (0x0e00 << 16) | (0x89bc >> 2),
 153         0x00000000,
 154         (0x0e00 << 16) | (0x8900 >> 2),
 155         0x00000000,
 156         0x3,
 157         (0x0e00 << 16) | (0xc130 >> 2),
 158         0x00000000,
 159         (0x0e00 << 16) | (0xc134 >> 2),
 160         0x00000000,
 161         (0x0e00 << 16) | (0xc1fc >> 2),
 162         0x00000000,
 163         (0x0e00 << 16) | (0xc208 >> 2),
 164         0x00000000,
 165         (0x0e00 << 16) | (0xc264 >> 2),
 166         0x00000000,
 167         (0x0e00 << 16) | (0xc268 >> 2),
 168         0x00000000,
 169         (0x0e00 << 16) | (0xc26c >> 2),
 170         0x00000000,
 171         (0x0e00 << 16) | (0xc270 >> 2),
 172         0x00000000,
 173         (0x0e00 << 16) | (0xc274 >> 2),
 174         0x00000000,
 175         (0x0e00 << 16) | (0xc278 >> 2),
 176         0x00000000,
 177         (0x0e00 << 16) | (0xc27c >> 2),
 178         0x00000000,
 179         (0x0e00 << 16) | (0xc280 >> 2),
 180         0x00000000,
 181         (0x0e00 << 16) | (0xc284 >> 2),
 182         0x00000000,
 183         (0x0e00 << 16) | (0xc288 >> 2),
 184         0x00000000,
 185         (0x0e00 << 16) | (0xc28c >> 2),
 186         0x00000000,
 187         (0x0e00 << 16) | (0xc290 >> 2),
 188         0x00000000,
 189         (0x0e00 << 16) | (0xc294 >> 2),
 190         0x00000000,
 191         (0x0e00 << 16) | (0xc298 >> 2),
 192         0x00000000,
 193         (0x0e00 << 16) | (0xc29c >> 2),
 194         0x00000000,
 195         (0x0e00 << 16) | (0xc2a0 >> 2),
 196         0x00000000,
 197         (0x0e00 << 16) | (0xc2a4 >> 2),
 198         0x00000000,
 199         (0x0e00 << 16) | (0xc2a8 >> 2),
 200         0x00000000,
 201         (0x0e00 << 16) | (0xc2ac  >> 2),
 202         0x00000000,
 203         (0x0e00 << 16) | (0xc2b0 >> 2),
 204         0x00000000,
 205         (0x0e00 << 16) | (0x301d0 >> 2),
 206         0x00000000,
 207         (0x0e00 << 16) | (0x30238 >> 2),
 208         0x00000000,
 209         (0x0e00 << 16) | (0x30250 >> 2),
 210         0x00000000,
 211         (0x0e00 << 16) | (0x30254 >> 2),
 212         0x00000000,
 213         (0x0e00 << 16) | (0x30258 >> 2),
 214         0x00000000,
 215         (0x0e00 << 16) | (0x3025c >> 2),
 216         0x00000000,
 217         (0x4e00 << 16) | (0xc900 >> 2),
 218         0x00000000,
 219         (0x5e00 << 16) | (0xc900 >> 2),
 220         0x00000000,
 221         (0x6e00 << 16) | (0xc900 >> 2),
 222         0x00000000,
 223         (0x7e00 << 16) | (0xc900 >> 2),
 224         0x00000000,
 225         (0x8e00 << 16) | (0xc900 >> 2),
 226         0x00000000,
 227         (0x9e00 << 16) | (0xc900 >> 2),
 228         0x00000000,
 229         (0xae00 << 16) | (0xc900 >> 2),
 230         0x00000000,
 231         (0xbe00 << 16) | (0xc900 >> 2),
 232         0x00000000,
 233         (0x4e00 << 16) | (0xc904 >> 2),
 234         0x00000000,
 235         (0x5e00 << 16) | (0xc904 >> 2),
 236         0x00000000,
 237         (0x6e00 << 16) | (0xc904 >> 2),
 238         0x00000000,
 239         (0x7e00 << 16) | (0xc904 >> 2),
 240         0x00000000,
 241         (0x8e00 << 16) | (0xc904 >> 2),
 242         0x00000000,
 243         (0x9e00 << 16) | (0xc904 >> 2),
 244         0x00000000,
 245         (0xae00 << 16) | (0xc904 >> 2),
 246         0x00000000,
 247         (0xbe00 << 16) | (0xc904 >> 2),
 248         0x00000000,
 249         (0x4e00 << 16) | (0xc908 >> 2),
 250         0x00000000,
 251         (0x5e00 << 16) | (0xc908 >> 2),
 252         0x00000000,
 253         (0x6e00 << 16) | (0xc908 >> 2),
 254         0x00000000,
 255         (0x7e00 << 16) | (0xc908 >> 2),
 256         0x00000000,
 257         (0x8e00 << 16) | (0xc908 >> 2),
 258         0x00000000,
 259         (0x9e00 << 16) | (0xc908 >> 2),
 260         0x00000000,
 261         (0xae00 << 16) | (0xc908 >> 2),
 262         0x00000000,
 263         (0xbe00 << 16) | (0xc908 >> 2),
 264         0x00000000,
 265         (0x4e00 << 16) | (0xc90c >> 2),
 266         0x00000000,
 267         (0x5e00 << 16) | (0xc90c >> 2),
 268         0x00000000,
 269         (0x6e00 << 16) | (0xc90c >> 2),
 270         0x00000000,
 271         (0x7e00 << 16) | (0xc90c >> 2),
 272         0x00000000,
 273         (0x8e00 << 16) | (0xc90c >> 2),
 274         0x00000000,
 275         (0x9e00 << 16) | (0xc90c >> 2),
 276         0x00000000,
 277         (0xae00 << 16) | (0xc90c >> 2),
 278         0x00000000,
 279         (0xbe00 << 16) | (0xc90c >> 2),
 280         0x00000000,
 281         (0x4e00 << 16) | (0xc910 >> 2),
 282         0x00000000,
 283         (0x5e00 << 16) | (0xc910 >> 2),
 284         0x00000000,
 285         (0x6e00 << 16) | (0xc910 >> 2),
 286         0x00000000,
 287         (0x7e00 << 16) | (0xc910 >> 2),
 288         0x00000000,
 289         (0x8e00 << 16) | (0xc910 >> 2),
 290         0x00000000,
 291         (0x9e00 << 16) | (0xc910 >> 2),
 292         0x00000000,
 293         (0xae00 << 16) | (0xc910 >> 2),
 294         0x00000000,
 295         (0xbe00 << 16) | (0xc910 >> 2),
 296         0x00000000,
 297         (0x0e00 << 16) | (0xc99c >> 2),
 298         0x00000000,
 299         (0x0e00 << 16) | (0x9834 >> 2),
 300         0x00000000,
 301         (0x0000 << 16) | (0x30f00 >> 2),
 302         0x00000000,
 303         (0x0001 << 16) | (0x30f00 >> 2),
 304         0x00000000,
 305         (0x0000 << 16) | (0x30f04 >> 2),
 306         0x00000000,
 307         (0x0001 << 16) | (0x30f04 >> 2),
 308         0x00000000,
 309         (0x0000 << 16) | (0x30f08 >> 2),
 310         0x00000000,
 311         (0x0001 << 16) | (0x30f08 >> 2),
 312         0x00000000,
 313         (0x0000 << 16) | (0x30f0c >> 2),
 314         0x00000000,
 315         (0x0001 << 16) | (0x30f0c >> 2),
 316         0x00000000,
 317         (0x0600 << 16) | (0x9b7c >> 2),
 318         0x00000000,
 319         (0x0e00 << 16) | (0x8a14 >> 2),
 320         0x00000000,
 321         (0x0e00 << 16) | (0x8a18 >> 2),
 322         0x00000000,
 323         (0x0600 << 16) | (0x30a00 >> 2),
 324         0x00000000,
 325         (0x0e00 << 16) | (0x8bf0 >> 2),
 326         0x00000000,
 327         (0x0e00 << 16) | (0x8bcc >> 2),
 328         0x00000000,
 329         (0x0e00 << 16) | (0x8b24 >> 2),
 330         0x00000000,
 331         (0x0e00 << 16) | (0x30a04 >> 2),
 332         0x00000000,
 333         (0x0600 << 16) | (0x30a10 >> 2),
 334         0x00000000,
 335         (0x0600 << 16) | (0x30a14 >> 2),
 336         0x00000000,
 337         (0x0600 << 16) | (0x30a18 >> 2),
 338         0x00000000,
 339         (0x0600 << 16) | (0x30a2c >> 2),
 340         0x00000000,
 341         (0x0e00 << 16) | (0xc700 >> 2),
 342         0x00000000,
 343         (0x0e00 << 16) | (0xc704 >> 2),
 344         0x00000000,
 345         (0x0e00 << 16) | (0xc708 >> 2),
 346         0x00000000,
 347         (0x0e00 << 16) | (0xc768 >> 2),
 348         0x00000000,
 349         (0x0400 << 16) | (0xc770 >> 2),
 350         0x00000000,
 351         (0x0400 << 16) | (0xc774 >> 2),
 352         0x00000000,
 353         (0x0400 << 16) | (0xc778 >> 2),
 354         0x00000000,
 355         (0x0400 << 16) | (0xc77c >> 2),
 356         0x00000000,
 357         (0x0400 << 16) | (0xc780 >> 2),
 358         0x00000000,
 359         (0x0400 << 16) | (0xc784 >> 2),
 360         0x00000000,
 361         (0x0400 << 16) | (0xc788 >> 2),
 362         0x00000000,
 363         (0x0400 << 16) | (0xc78c >> 2),
 364         0x00000000,
 365         (0x0400 << 16) | (0xc798 >> 2),
 366         0x00000000,
 367         (0x0400 << 16) | (0xc79c >> 2),
 368         0x00000000,
 369         (0x0400 << 16) | (0xc7a0 >> 2),
 370         0x00000000,
 371         (0x0400 << 16) | (0xc7a4 >> 2),
 372         0x00000000,
 373         (0x0400 << 16) | (0xc7a8 >> 2),
 374         0x00000000,
 375         (0x0400 << 16) | (0xc7ac >> 2),
 376         0x00000000,
 377         (0x0400 << 16) | (0xc7b0 >> 2),
 378         0x00000000,
 379         (0x0400 << 16) | (0xc7b4 >> 2),
 380         0x00000000,
 381         (0x0e00 << 16) | (0x9100 >> 2),
 382         0x00000000,
 383         (0x0e00 << 16) | (0x3c010 >> 2),
 384         0x00000000,
 385         (0x0e00 << 16) | (0x92a8 >> 2),
 386         0x00000000,
 387         (0x0e00 << 16) | (0x92ac >> 2),
 388         0x00000000,
 389         (0x0e00 << 16) | (0x92b4 >> 2),
 390         0x00000000,
 391         (0x0e00 << 16) | (0x92b8 >> 2),
 392         0x00000000,
 393         (0x0e00 << 16) | (0x92bc >> 2),
 394         0x00000000,
 395         (0x0e00 << 16) | (0x92c0 >> 2),
 396         0x00000000,
 397         (0x0e00 << 16) | (0x92c4 >> 2),
 398         0x00000000,
 399         (0x0e00 << 16) | (0x92c8 >> 2),
 400         0x00000000,
 401         (0x0e00 << 16) | (0x92cc >> 2),
 402         0x00000000,
 403         (0x0e00 << 16) | (0x92d0 >> 2),
 404         0x00000000,
 405         (0x0e00 << 16) | (0x8c00 >> 2),
 406         0x00000000,
 407         (0x0e00 << 16) | (0x8c04 >> 2),
 408         0x00000000,
 409         (0x0e00 << 16) | (0x8c20 >> 2),
 410         0x00000000,
 411         (0x0e00 << 16) | (0x8c38 >> 2),
 412         0x00000000,
 413         (0x0e00 << 16) | (0x8c3c >> 2),
 414         0x00000000,
 415         (0x0e00 << 16) | (0xae00 >> 2),
 416         0x00000000,
 417         (0x0e00 << 16) | (0x9604 >> 2),
 418         0x00000000,
 419         (0x0e00 << 16) | (0xac08 >> 2),
 420         0x00000000,
 421         (0x0e00 << 16) | (0xac0c >> 2),
 422         0x00000000,
 423         (0x0e00 << 16) | (0xac10 >> 2),
 424         0x00000000,
 425         (0x0e00 << 16) | (0xac14 >> 2),
 426         0x00000000,
 427         (0x0e00 << 16) | (0xac58 >> 2),
 428         0x00000000,
 429         (0x0e00 << 16) | (0xac68 >> 2),
 430         0x00000000,
 431         (0x0e00 << 16) | (0xac6c >> 2),
 432         0x00000000,
 433         (0x0e00 << 16) | (0xac70 >> 2),
 434         0x00000000,
 435         (0x0e00 << 16) | (0xac74 >> 2),
 436         0x00000000,
 437         (0x0e00 << 16) | (0xac78 >> 2),
 438         0x00000000,
 439         (0x0e00 << 16) | (0xac7c >> 2),
 440         0x00000000,
 441         (0x0e00 << 16) | (0xac80 >> 2),
 442         0x00000000,
 443         (0x0e00 << 16) | (0xac84 >> 2),
 444         0x00000000,
 445         (0x0e00 << 16) | (0xac88 >> 2),
 446         0x00000000,
 447         (0x0e00 << 16) | (0xac8c >> 2),
 448         0x00000000,
 449         (0x0e00 << 16) | (0x970c >> 2),
 450         0x00000000,
 451         (0x0e00 << 16) | (0x9714 >> 2),
 452         0x00000000,
 453         (0x0e00 << 16) | (0x9718 >> 2),
 454         0x00000000,
 455         (0x0e00 << 16) | (0x971c >> 2),
 456         0x00000000,
 457         (0x0e00 << 16) | (0x31068 >> 2),
 458         0x00000000,
 459         (0x4e00 << 16) | (0x31068 >> 2),
 460         0x00000000,
 461         (0x5e00 << 16) | (0x31068 >> 2),
 462         0x00000000,
 463         (0x6e00 << 16) | (0x31068 >> 2),
 464         0x00000000,
 465         (0x7e00 << 16) | (0x31068 >> 2),
 466         0x00000000,
 467         (0x8e00 << 16) | (0x31068 >> 2),
 468         0x00000000,
 469         (0x9e00 << 16) | (0x31068 >> 2),
 470         0x00000000,
 471         (0xae00 << 16) | (0x31068 >> 2),
 472         0x00000000,
 473         (0xbe00 << 16) | (0x31068 >> 2),
 474         0x00000000,
 475         (0x0e00 << 16) | (0xcd10 >> 2),
 476         0x00000000,
 477         (0x0e00 << 16) | (0xcd14 >> 2),
 478         0x00000000,
 479         (0x0e00 << 16) | (0x88b0 >> 2),
 480         0x00000000,
 481         (0x0e00 << 16) | (0x88b4 >> 2),
 482         0x00000000,
 483         (0x0e00 << 16) | (0x88b8 >> 2),
 484         0x00000000,
 485         (0x0e00 << 16) | (0x88bc >> 2),
 486         0x00000000,
 487         (0x0400 << 16) | (0x89c0 >> 2),
 488         0x00000000,
 489         (0x0e00 << 16) | (0x88c4 >> 2),
 490         0x00000000,
 491         (0x0e00 << 16) | (0x88c8 >> 2),
 492         0x00000000,
 493         (0x0e00 << 16) | (0x88d0 >> 2),
 494         0x00000000,
 495         (0x0e00 << 16) | (0x88d4 >> 2),
 496         0x00000000,
 497         (0x0e00 << 16) | (0x88d8 >> 2),
 498         0x00000000,
 499         (0x0e00 << 16) | (0x8980 >> 2),
 500         0x00000000,
 501         (0x0e00 << 16) | (0x30938 >> 2),
 502         0x00000000,
 503         (0x0e00 << 16) | (0x3093c >> 2),
 504         0x00000000,
 505         (0x0e00 << 16) | (0x30940 >> 2),
 506         0x00000000,
 507         (0x0e00 << 16) | (0x89a0 >> 2),
 508         0x00000000,
 509         (0x0e00 << 16) | (0x30900 >> 2),
 510         0x00000000,
 511         (0x0e00 << 16) | (0x30904 >> 2),
 512         0x00000000,
 513         (0x0e00 << 16) | (0x89b4 >> 2),
 514         0x00000000,
 515         (0x0e00 << 16) | (0x3c210 >> 2),
 516         0x00000000,
 517         (0x0e00 << 16) | (0x3c214 >> 2),
 518         0x00000000,
 519         (0x0e00 << 16) | (0x3c218 >> 2),
 520         0x00000000,
 521         (0x0e00 << 16) | (0x8904 >> 2),
 522         0x00000000,
 523         0x5,
 524         (0x0e00 << 16) | (0x8c28 >> 2),
 525         (0x0e00 << 16) | (0x8c2c >> 2),
 526         (0x0e00 << 16) | (0x8c30 >> 2),
 527         (0x0e00 << 16) | (0x8c34 >> 2),
 528         (0x0e00 << 16) | (0x9600 >> 2),
 529 };
 530
 531 static const u32 kalindi_rlc_save_restore_register_list[] =
 532 {
 533         (0x0e00 << 16) | (0xc12c >> 2),
 534         0x00000000,
 535         (0x0e00 << 16) | (0xc140 >> 2),
 536         0x00000000,
 537         (0x0e00 << 16) | (0xc150 >> 2),
 538         0x00000000,
 539         (0x0e00 << 16) | (0xc15c >> 2),
 540         0x00000000,
 541         (0x0e00 << 16) | (0xc168 >> 2),
 542         0x00000000,
 543         (0x0e00 << 16) | (0xc170 >> 2),
 544         0x00000000,
 545         (0x0e00 << 16) | (0xc204 >> 2),
 546         0x00000000,
 547         (0x0e00 << 16) | (0xc2b4 >> 2),
 548         0x00000000,
 549         (0x0e00 << 16) | (0xc2b8 >> 2),
 550         0x00000000,
 551         (0x0e00 << 16) | (0xc2bc >> 2),
 552         0x00000000,
 553         (0x0e00 << 16) | (0xc2c0 >> 2),
 554         0x00000000,
 555         (0x0e00 << 16) | (0x8228 >> 2),
 556         0x00000000,
 557         (0x0e00 << 16) | (0x829c >> 2),
 558         0x00000000,
 559         (0x0e00 << 16) | (0x869c >> 2),
 560         0x00000000,
 561         (0x0600 << 16) | (0x98f4 >> 2),
 562         0x00000000,
 563         (0x0e00 << 16) | (0x98f8 >> 2),
 564         0x00000000,
 565         (0x0e00 << 16) | (0x9900 >> 2),
 566         0x00000000,
 567         (0x0e00 << 16) | (0xc260 >> 2),
 568         0x00000000,
 569         (0x0e00 << 16) | (0x90e8 >> 2),
 570         0x00000000,
 571         (0x0e00 << 16) | (0x3c000 >> 2),
 572         0x00000000,
 573         (0x0e00 << 16) | (0x3c00c >> 2),
 574         0x00000000,
 575         (0x0e00 << 16) | (0x8c1c >> 2),
 576         0x00000000,
 577         (0x0e00 << 16) | (0x9700 >> 2),
 578         0x00000000,
 579         (0x0e00 << 16) | (0xcd20 >> 2),
 580         0x00000000,
 581         (0x4e00 << 16) | (0xcd20 >> 2),
 582         0x00000000,
 583         (0x5e00 << 16) | (0xcd20 >> 2),
 584         0x00000000,
 585         (0x6e00 << 16) | (0xcd20 >> 2),
 586         0x00000000,
 587         (0x7e00 << 16) | (0xcd20 >> 2),
 588         0x00000000,
 589         (0x0e00 << 16) | (0x89bc >> 2),
 590         0x00000000,
 591         (0x0e00 << 16) | (0x8900 >> 2),
 592         0x00000000,
 593         0x3,
 594         (0x0e00 << 16) | (0xc130 >> 2),
 595         0x00000000,
 596         (0x0e00 << 16) | (0xc134 >> 2),
 597         0x00000000,
 598         (0x0e00 << 16) | (0xc1fc >> 2),
 599         0x00000000,
 600         (0x0e00 << 16) | (0xc208 >> 2),
 601         0x00000000,
 602         (0x0e00 << 16) | (0xc264 >> 2),
 603         0x00000000,
 604         (0x0e00 << 16) | (0xc268 >> 2),
 605         0x00000000,
 606         (0x0e00 << 16) | (0xc26c >> 2),
 607         0x00000000,
 608         (0x0e00 << 16) | (0xc270 >> 2),
 609         0x00000000,
 610         (0x0e00 << 16) | (0xc274 >> 2),
 611         0x00000000,
 612         (0x0e00 << 16) | (0xc28c >> 2),
 613         0x00000000,
 614         (0x0e00 << 16) | (0xc290 >> 2),
 615         0x00000000,
 616         (0x0e00 << 16) | (0xc294 >> 2),
 617         0x00000000,
 618         (0x0e00 << 16) | (0xc298 >> 2),
 619         0x00000000,
 620         (0x0e00 << 16) | (0xc2a0 >> 2),
 621         0x00000000,
 622         (0x0e00 << 16) | (0xc2a4 >> 2),
 623         0x00000000,
 624         (0x0e00 << 16) | (0xc2a8 >> 2),
 625         0x00000000,
 626         (0x0e00 << 16) | (0xc2ac >> 2),
 627         0x00000000,
 628         (0x0e00 << 16) | (0x301d0 >> 2),
 629         0x00000000,
 630         (0x0e00 << 16) | (0x30238 >> 2),
 631         0x00000000,
 632         (0x0e00 << 16) | (0x30250 >> 2),
 633         0x00000000,
 634         (0x0e00 << 16) | (0x30254 >> 2),
 635         0x00000000,
 636         (0x0e00 << 16) | (0x30258 >> 2),
 637         0x00000000,
 638         (0x0e00 << 16) | (0x3025c >> 2),
 639         0x00000000,
 640         (0x4e00 << 16) | (0xc900 >> 2),
 641         0x00000000,
 642         (0x5e00 << 16) | (0xc900 >> 2),
 643         0x00000000,
 644         (0x6e00 << 16) | (0xc900 >> 2),
 645         0x00000000,
 646         (0x7e00 << 16) | (0xc900 >> 2),
 647         0x00000000,
 648         (0x4e00 << 16) | (0xc904 >> 2),
 649         0x00000000,
 650         (0x5e00 << 16) | (0xc904 >> 2),
 651         0x00000000,
 652         (0x6e00 << 16) | (0xc904 >> 2),
 653         0x00000000,
 654         (0x7e00 << 16) | (0xc904 >> 2),
 655         0x00000000,
 656         (0x4e00 << 16) | (0xc908 >> 2),
 657         0x00000000,
 658         (0x5e00 << 16) | (0xc908 >> 2),
 659         0x00000000,
 660         (0x6e00 << 16) | (0xc908 >> 2),
 661         0x00000000,
 662         (0x7e00 << 16) | (0xc908 >> 2),
 663         0x00000000,
 664         (0x4e00 << 16) | (0xc90c >> 2),
 665         0x00000000,
 666         (0x5e00 << 16) | (0xc90c >> 2),
 667         0x00000000,
 668         (0x6e00 << 16) | (0xc90c >> 2),
 669         0x00000000,
 670         (0x7e00 << 16) | (0xc90c >> 2),
 671         0x00000000,
 672         (0x4e00 << 16) | (0xc910 >> 2),
 673         0x00000000,
 674         (0x5e00 << 16) | (0xc910 >> 2),
 675         0x00000000,
 676         (0x6e00 << 16) | (0xc910 >> 2),
 677         0x00000000,
 678         (0x7e00 << 16) | (0xc910 >> 2),
 679         0x00000000,
 680         (0x0e00 << 16) | (0xc99c >> 2),
 681         0x00000000,
 682         (0x0e00 << 16) | (0x9834 >> 2),
 683         0x00000000,
 684         (0x0000 << 16) | (0x30f00 >> 2),
 685         0x00000000,
 686         (0x0000 << 16) | (0x30f04 >> 2),
 687         0x00000000,
 688         (0x0000 << 16) | (0x30f08 >> 2),
 689         0x00000000,
 690         (0x0000 << 16) | (0x30f0c >> 2),
 691         0x00000000,
 692         (0x0600 << 16) | (0x9b7c >> 2),
 693         0x00000000,
 694         (0x0e00 << 16) | (0x8a14 >> 2),
 695         0x00000000,
 696         (0x0e00 << 16) | (0x8a18 >> 2),
 697         0x00000000,
 698         (0x0600 << 16) | (0x30a00 >> 2),
 699         0x00000000,
 700         (0x0e00 << 16) | (0x8bf0 >> 2),
 701         0x00000000,
 702         (0x0e00 << 16) | (0x8bcc >> 2),
 703         0x00000000,
 704         (0x0e00 << 16) | (0x8b24 >> 2),
 705         0x00000000,
 706         (0x0e00 << 16) | (0x30a04 >> 2),
 707         0x00000000,
 708         (0x0600 << 16) | (0x30a10 >> 2),
 709         0x00000000,
 710         (0x0600 << 16) | (0x30a14 >> 2),
 711         0x00000000,
 712         (0x0600 << 16) | (0x30a18 >> 2),
 713         0x00000000,
 714         (0x0600 << 16) | (0x30a2c >> 2),
 715         0x00000000,
 716         (0x0e00 << 16) | (0xc700 >> 2),
 717         0x00000000,
 718         (0x0e00 << 16) | (0xc704 >> 2),
 719         0x00000000,
 720         (0x0e00 << 16) | (0xc708 >> 2),
 721         0x00000000,
 722         (0x0e00 << 16) | (0xc768 >> 2),
 723         0x00000000,
 724         (0x0400 << 16) | (0xc770 >> 2),
 725         0x00000000,
 726         (0x0400 << 16) | (0xc774 >> 2),
 727         0x00000000,
 728         (0x0400 << 16) | (0xc798 >> 2),
 729         0x00000000,
 730         (0x0400 << 16) | (0xc79c >> 2),
 731         0x00000000,
 732         (0x0e00 << 16) | (0x9100 >> 2),
 733         0x00000000,
 734         (0x0e00 << 16) | (0x3c010 >> 2),
 735         0x00000000,
 736         (0x0e00 << 16) | (0x8c00 >> 2),
 737         0x00000000,
 738         (0x0e00 << 16) | (0x8c04 >> 2),
 739         0x00000000,
 740         (0x0e00 << 16) | (0x8c20 >> 2),
 741         0x00000000,
 742         (0x0e00 << 16) | (0x8c38 >> 2),
 743         0x00000000,
 744         (0x0e00 << 16) | (0x8c3c >> 2),
 745         0x00000000,
 746         (0x0e00 << 16) | (0xae00 >> 2),
 747         0x00000000,
 748         (0x0e00 << 16) | (0x9604 >> 2),
 749         0x00000000,
 750         (0x0e00 << 16) | (0xac08 >> 2),
 751         0x00000000,
 752         (0x0e00 << 16) | (0xac0c >> 2),
 753         0x00000000,
 754         (0x0e00 << 16) | (0xac10 >> 2),
 755         0x00000000,
 756         (0x0e00 << 16) | (0xac14 >> 2),
 757         0x00000000,
 758         (0x0e00 << 16) | (0xac58 >> 2),
 759         0x00000000,
 760         (0x0e00 << 16) | (0xac68 >> 2),
 761         0x00000000,
 762         (0x0e00 << 16) | (0xac6c >> 2),
 763         0x00000000,
 764         (0x0e00 << 16) | (0xac70 >> 2),
 765         0x00000000,
 766         (0x0e00 << 16) | (0xac74 >> 2),
 767         0x00000000,
 768         (0x0e00 << 16) | (0xac78 >> 2),
 769         0x00000000,
 770         (0x0e00 << 16) | (0xac7c >> 2),
 771         0x00000000,
 772         (0x0e00 << 16) | (0xac80 >> 2),
 773         0x00000000,
 774         (0x0e00 << 16) | (0xac84 >> 2),
 775         0x00000000,
 776         (0x0e00 << 16) | (0xac88 >> 2),
 777         0x00000000,
 778         (0x0e00 << 16) | (0xac8c >> 2),
 779         0x00000000,
 780         (0x0e00 << 16) | (0x970c >> 2),
 781         0x00000000,
 782         (0x0e00 << 16) | (0x9714 >> 2),
 783         0x00000000,
 784         (0x0e00 << 16) | (0x9718 >> 2),
 785         0x00000000,
 786         (0x0e00 << 16) | (0x971c >> 2),
 787         0x00000000,
 788         (0x0e00 << 16) | (0x31068 >> 2),
 789         0x00000000,
 790         (0x4e00 << 16) | (0x31068 >> 2),
 791         0x00000000,
 792         (0x5e00 << 16) | (0x31068 >> 2),
 793         0x00000000,
 794         (0x6e00 << 16) | (0x31068 >> 2),
 795         0x00000000,
 796         (0x7e00 << 16) | (0x31068 >> 2),
 797         0x00000000,
 798         (0x0e00 << 16) | (0xcd10 >> 2),
 799         0x00000000,
 800         (0x0e00 << 16) | (0xcd14 >> 2),
 801         0x00000000,
 802         (0x0e00 << 16) | (0x88b0 >> 2),
 803         0x00000000,
 804         (0x0e00 << 16) | (0x88b4 >> 2),
 805         0x00000000,
 806         (0x0e00 << 16) | (0x88b8 >> 2),
 807         0x00000000,
 808         (0x0e00 << 16) | (0x88bc >> 2),
 809         0x00000000,
 810         (0x0400 << 16) | (0x89c0 >> 2),
 811         0x00000000,
 812         (0x0e00 << 16) | (0x88c4 >> 2),
 813         0x00000000,
 814         (0x0e00 << 16) | (0x88c8 >> 2),
 815         0x00000000,
 816         (0x0e00 << 16) | (0x88d0 >> 2),
 817         0x00000000,
 818         (0x0e00 << 16) | (0x88d4 >> 2),
 819         0x00000000,
 820         (0x0e00 << 16) | (0x88d8 >> 2),
 821         0x00000000,
 822         (0x0e00 << 16) | (0x8980 >> 2),
 823         0x00000000,
 824         (0x0e00 << 16) | (0x30938 >> 2),
 825         0x00000000,
 826         (0x0e00 << 16) | (0x3093c >> 2),
 827         0x00000000,
 828         (0x0e00 << 16) | (0x30940 >> 2),
 829         0x00000000,
 830         (0x0e00 << 16) | (0x89a0 >> 2),
 831         0x00000000,
 832         (0x0e00 << 16) | (0x30900 >> 2),
 833         0x00000000,
 834         (0x0e00 << 16) | (0x30904 >> 2),
 835         0x00000000,
 836         (0x0e00 << 16) | (0x89b4 >> 2),
 837         0x00000000,
 838         (0x0e00 << 16) | (0x3e1fc >> 2),
 839         0x00000000,
 840         (0x0e00 << 16) | (0x3c210 >> 2),
 841         0x00000000,
 842         (0x0e00 << 16) | (0x3c214 >> 2),
 843         0x00000000,
 844         (0x0e00 << 16) | (0x3c218 >> 2),
 845         0x00000000,
 846         (0x0e00 << 16) | (0x8904 >> 2),
 847         0x00000000,
 848         0x5,
 849         (0x0e00 << 16) | (0x8c28 >> 2),
 850         (0x0e00 << 16) | (0x8c2c >> 2),
 851         (0x0e00 << 16) | (0x8c30 >> 2),
 852         (0x0e00 << 16) | (0x8c34 >> 2),
 853         (0x0e00 << 16) | (0x9600 >> 2),
 854 };
 855
 856 static u32 gfx_v7_0_get_csb_size(struct amdgpu_device *adev);
 857 static void gfx_v7_0_get_csb_buffer(struct amdgpu_device *adev, volatile u32 *buffer);
 858 static void gfx_v7_0_init_pg(struct amdgpu_device *adev);
 859 static void gfx_v7_0_get_cu_info(struct amdgpu_device *adev);
 860
 861 /*
 862  * Core functions
 863  */
 864 /**
 865  * gfx_v7_0_init_microcode - load ucode images from disk
 866  *
 867  * @adev: amdgpu_device pointer
 868  *
 869  * Use the firmware interface to load the ucode images into
 870  * the driver (not loaded into hw).
 871  * Returns 0 on success, error on failure.
 872  */
 873 static int gfx_v7_0_init_microcode(struct amdgpu_device *adev)
 874 {
 875         const char *chip_name;
 876         char fw_name[30];
 877         int err;
 878
 879         DRM_DEBUG("\n");
 880
 881         switch (adev->asic_type) {
 882         case CHIP_BONAIRE:
 883                 chip_name = "bonaire";
 884                 break;
 885         case CHIP_HAWAII:
 886                 chip_name = "hawaii";
 887                 break;
 888         case CHIP_KAVERI:
 889                 chip_name = "kaveri";
 890                 break;
 891         case CHIP_KABINI:
 892                 chip_name = "kabini";
 893                 break;
 894         case CHIP_MULLINS:
 895                 chip_name = "mullins";
 896                 break;
 897         default: BUG();
 898         }
 899
 900         snprintf(fw_name, sizeof(fw_name), "/*(DEBLOBBED)*/", chip_name);
 901         err = reject_firmware(&adev->gfx.pfp_fw, fw_name, adev->dev);
 902         if (err)
 903                 goto out;
 904         err = amdgpu_ucode_validate(adev->gfx.pfp_fw);
 905         if (err)
 906                 goto out;
 907
 908         snprintf(fw_name, sizeof(fw_name), "/*(DEBLOBBED)*/", chip_name);
 909         err = reject_firmware(&adev->gfx.me_fw, fw_name, adev->dev);
 910         if (err)
 911                 goto out;
 912         err = amdgpu_ucode_validate(adev->gfx.me_fw);
 913         if (err)
 914                 goto out;
 915
 916         snprintf(fw_name, sizeof(fw_name), "/*(DEBLOBBED)*/", chip_name);
 917         err = reject_firmware(&adev->gfx.ce_fw, fw_name, adev->dev);
 918         if (err)
 919                 goto out;
 920         err = amdgpu_ucode_validate(adev->gfx.ce_fw);
 921         if (err)
 922                 goto out;
 923
 924         snprintf(fw_name, sizeof(fw_name), "/*(DEBLOBBED)*/", chip_name);
 925         err = reject_firmware(&adev->gfx.mec_fw, fw_name, adev->dev);
 926         if (err)
 927                 goto out;
 928         err = amdgpu_ucode_validate(adev->gfx.mec_fw);
 929         if (err)
 930                 goto out;
 931
 932         if (adev->asic_type == CHIP_KAVERI) {
 933                 snprintf(fw_name, sizeof(fw_name), "/*(DEBLOBBED)*/", chip_name);
 934                 err = reject_firmware(&adev->gfx.mec2_fw, fw_name, adev->dev);
 935                 if (err)
 936                         goto out;
 937                 err = amdgpu_ucode_validate(adev->gfx.mec2_fw);
 938                 if (err)
 939                         goto out;
 940         }
 941
 942         snprintf(fw_name, sizeof(fw_name), "/*(DEBLOBBED)*/", chip_name);
 943         err = reject_firmware(&adev->gfx.rlc_fw, fw_name, adev->dev);
 944         if (err)
 945                 goto out;
 946         err = amdgpu_ucode_validate(adev->gfx.rlc_fw);
 947
 948 out:
 949         if (err) {
 950                 pr_err("gfx7: Failed to load firmware \"%s\"\n", fw_name);
 951                 release_firmware(adev->gfx.pfp_fw);
 952                 adev->gfx.pfp_fw = NULL;
 953                 release_firmware(adev->gfx.me_fw);
 954                 adev->gfx.me_fw = NULL;
 955                 release_firmware(adev->gfx.ce_fw);
 956                 adev->gfx.ce_fw = NULL;
 957                 release_firmware(adev->gfx.mec_fw);
 958                 adev->gfx.mec_fw = NULL;
 959                 release_firmware(adev->gfx.mec2_fw);
 960                 adev->gfx.mec2_fw = NULL;
 961                 release_firmware(adev->gfx.rlc_fw);
 962                 adev->gfx.rlc_fw = NULL;
 963         }
 964         return err;
 965 }
 966
 967 static void gfx_v7_0_free_microcode(struct amdgpu_device *adev)
 968 {
 969         release_firmware(adev->gfx.pfp_fw);
 970         adev->gfx.pfp_fw = NULL;
 971         release_firmware(adev->gfx.me_fw);
 972         adev->gfx.me_fw = NULL;
 973         release_firmware(adev->gfx.ce_fw);
 974         adev->gfx.ce_fw = NULL;
 975         release_firmware(adev->gfx.mec_fw);
 976         adev->gfx.mec_fw = NULL;
 977         release_firmware(adev->gfx.mec2_fw);
 978         adev->gfx.mec2_fw = NULL;
 979         release_firmware(adev->gfx.rlc_fw);
 980         adev->gfx.rlc_fw = NULL;
 981 }
 982
 983 /**
 984  * gfx_v7_0_tiling_mode_table_init - init the hw tiling table
 985  *
 986  * @adev: amdgpu_device pointer
 987  *
 988  * Starting with SI, the tiling setup is done globally in a
 989  * set of 32 tiling modes.  Rather than selecting each set of
 990  * parameters per surface as on older asics, we just select
 991  * which index in the tiling table we want to use, and the
 992  * surface uses those parameters (CIK).
 993  */
 994 static void gfx_v7_0_tiling_mode_table_init(struct amdgpu_device *adev)
 995 {
 996         const u32 num_tile_mode_states =
 997                         ARRAY_SIZE(adev->gfx.config.tile_mode_array);
 998         const u32 num_secondary_tile_mode_states =
 999                         ARRAY_SIZE(adev->gfx.config.macrotile_mode_array);
1000         u32 reg_offset, split_equal_to_row_size;
1001         uint32_t *tile, *macrotile;
1002
1003         tile = adev->gfx.config.tile_mode_array;
1004         macrotile = adev->gfx.config.macrotile_mode_array;
1005
1006         switch (adev->gfx.config.mem_row_size_in_kb) {
1007         case 1:
1008                 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
1009                 break;
1010         case 2:
1011         default:
1012                 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
1013                 break;
1014         case 4:
1015                 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
1016                 break;
1017         }
1018
1019         for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
1020                 tile[reg_offset] = 0;
1021         for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
1022                 macrotile[reg_offset] = 0;
1023
1024         switch (adev->asic_type) {
1025         case CHIP_BONAIRE:
1026                 tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1027                            PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1028                            TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
1029                            MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
1030                 tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1031                            PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1032                            TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
1033                            MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
1034                 tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1035                            PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1036                            TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
1037                            MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
1038                 tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1039                            PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1040                            TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
1041                            MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
1042                 tile[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1043                            PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1044                            MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1045                            TILE_SPLIT(split_equal_to_row_size));
1046                 tile[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1047                            PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1048                            MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
1049                 tile[6] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
1050                            PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1051                            MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1052                            TILE_SPLIT(split_equal_to_row_size));
1053                 tile[7] = (TILE_SPLIT(split_equal_to_row_size));
1054                 tile[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
1055                            PIPE_CONFIG(ADDR_SURF_P4_16x16));
1056                 tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1057                            PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1058                            MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
1059                 tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1060                             PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1061                             MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
1062                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1063                 tile[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
1064                             PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1065                             MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
1066                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
1067                 tile[12] = (TILE_SPLIT(split_equal_to_row_size));
1068                 tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1069                             PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1070                             MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
1071                 tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1072                             PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1073                             MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1074                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1075                 tile[15] = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) |
1076                             PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1077                             MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1078                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1079                 tile[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
1080                             PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1081                             MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1082                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
1083                 tile[17] = (TILE_SPLIT(split_equal_to_row_size));
1084                 tile[18] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
1085                             PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1086                             MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1087                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
1088                 tile[19] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
1089                             PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1090                             MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
1091                 tile[20] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
1092                             PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1093                             MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1094                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
1095                 tile[21] =  (ARRAY_MODE(ARRAY_3D_TILED_THICK) |
1096                             PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1097                             MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1098                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
1099                 tile[22] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |
1100                             PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1101                             MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1102                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
1103                 tile[23] = (TILE_SPLIT(split_equal_to_row_size));
1104                 tile[24] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
1105                             PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1106                             MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1107                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
1108                 tile[25] = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) |
1109                             PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1110                             MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1111                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
1112                 tile[26] = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) |
1113                             PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1114                             MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1115                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
1116                 tile[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1117                             PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1118                             MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
1119                 tile[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1120                             PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1121                             MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
1122                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1123                 tile[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
1124                             PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1125                             MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
1126                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
1127                 tile[30] = (TILE_SPLIT(split_equal_to_row_size));
1128
1129                 macrotile[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1130                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1131                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1132                                 NUM_BANKS(ADDR_SURF_16_BANK));
1133                 macrotile[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1134                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1135                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1136                                 NUM_BANKS(ADDR_SURF_16_BANK));
1137                 macrotile[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1138                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1139                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1140                                 NUM_BANKS(ADDR_SURF_16_BANK));
1141                 macrotile[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1142                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1143                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1144                                 NUM_BANKS(ADDR_SURF_16_BANK));
1145                 macrotile[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1146                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1147                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1148                                 NUM_BANKS(ADDR_SURF_16_BANK));
1149                 macrotile[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1150                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1151                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1152                                 NUM_BANKS(ADDR_SURF_8_BANK));
1153                 macrotile[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1154                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1155                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1156                                 NUM_BANKS(ADDR_SURF_4_BANK));
1157                 macrotile[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
1158                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
1159                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1160                                 NUM_BANKS(ADDR_SURF_16_BANK));
1161                 macrotile[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
1162                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1163                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1164                                 NUM_BANKS(ADDR_SURF_16_BANK));
1165                 macrotile[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1166                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1167                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1168                                 NUM_BANKS(ADDR_SURF_16_BANK));
1169                 macrotile[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1170                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1171                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1172                                 NUM_BANKS(ADDR_SURF_16_BANK));
1173                 macrotile[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1174                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1175                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1176                                 NUM_BANKS(ADDR_SURF_16_BANK));
1177                 macrotile[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1178                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1179                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1180                                 NUM_BANKS(ADDR_SURF_8_BANK));
1181                 macrotile[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1182                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1183                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1184                                 NUM_BANKS(ADDR_SURF_4_BANK));
1185
1186                 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
1187                         WREG32(mmGB_TILE_MODE0 + reg_offset, tile[reg_offset]);
1188                 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
1189                         if (reg_offset != 7)
1190                                 WREG32(mmGB_MACROTILE_MODE0 + reg_offset, macrotile[reg_offset]);
1191                 break;
1192         case CHIP_HAWAII:
1193                 tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1194                            PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
1195                            TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
1196                            MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
1197                 tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1198                            PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
1199                            TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
1200                            MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
1201                 tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1202                            PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
1203                            TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
1204                            MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
1205                 tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1206                            PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
1207                            TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
1208                            MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
1209                 tile[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1210                            PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
1211                            MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1212                            TILE_SPLIT(split_equal_to_row_size));
1213                 tile[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1214                            PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
1215                            MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1216                            TILE_SPLIT(split_equal_to_row_size));
1217                 tile[6] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
1218                            PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
1219                            MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1220                            TILE_SPLIT(split_equal_to_row_size));
1221                 tile[7] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
1222                            PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1223                            MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1224                            TILE_SPLIT(split_equal_to_row_size));
1225                 tile[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
1226                            PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16));
1227                 tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1228                            PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
1229                            MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
1230                 tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1231                             PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
1232                             MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
1233                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1234                 tile[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
1235                             PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
1236                             MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
1237                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
1238                 tile[12] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
1239                             PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1240                             MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
1241                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
1242                 tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1243                             PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
1244                             MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
1245                 tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1246                             PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
1247                             MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1248                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1249                 tile[15] = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) |
1250                             PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
1251                             MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1252                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1253                 tile[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
1254                             PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
1255                             MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1256                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
1257                 tile[17] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
1258                             PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1259                             MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1260                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
1261                 tile[18] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
1262                             PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
1263                             MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1264                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
1265                 tile[19] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
1266                             PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
1267                             MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING));
1268                 tile[20] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
1269                             PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
1270                             MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
1271                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
1272                 tile[21] = (ARRAY_MODE(ARRAY_3D_TILED_THICK) |
1273                             PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
1274                             MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
1275                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
1276                 tile[22] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |
1277                             PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
1278                             MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
1279                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
1280                 tile[23] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |
1281                             PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1282                             MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
1283                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
1284                 tile[24] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
1285                             PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
1286                             MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1287                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
1288                 tile[25] = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) |
1289                             PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
1290                             MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
1291                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
1292                 tile[26] = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) |
1293                             PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
1294                             MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
1295                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
1296                 tile[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1297                             PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
1298                             MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
1299                 tile[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1300                             PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
1301                             MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
1302                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1303                 tile[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
1304                             PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
1305                             MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
1306                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
1307                 tile[30] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
1308                             PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1309                             MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
1310                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
1311
1312                 macrotile[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1313                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1314                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1315                                 NUM_BANKS(ADDR_SURF_16_BANK));
1316                 macrotile[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1317                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1318                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1319                                 NUM_BANKS(ADDR_SURF_16_BANK));
1320                 macrotile[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1321                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1322                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1323                                 NUM_BANKS(ADDR_SURF_16_BANK));
1324                 macrotile[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1325                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1326                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1327                                 NUM_BANKS(ADDR_SURF_16_BANK));
1328                 macrotile[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1329                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1330                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1331                                 NUM_BANKS(ADDR_SURF_8_BANK));
1332                 macrotile[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1333                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1334                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1335                                 NUM_BANKS(ADDR_SURF_4_BANK));
1336                 macrotile[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1337                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1338                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1339                                 NUM_BANKS(ADDR_SURF_4_BANK));
1340                 macrotile[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1341                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1342                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1343                                 NUM_BANKS(ADDR_SURF_16_BANK));
1344                 macrotile[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1345                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1346                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1347                                 NUM_BANKS(ADDR_SURF_16_BANK));
1348                 macrotile[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1349                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1350                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1351                                 NUM_BANKS(ADDR_SURF_16_BANK));
1352                 macrotile[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1353                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1354                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1355                                 NUM_BANKS(ADDR_SURF_8_BANK));
1356                 macrotile[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1357                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1358                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1359                                 NUM_BANKS(ADDR_SURF_16_BANK));
1360                 macrotile[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1361                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1362                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1363                                 NUM_BANKS(ADDR_SURF_8_BANK));
1364                 macrotile[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1365                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1366                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1367                                 NUM_BANKS(ADDR_SURF_4_BANK));
1368
1369                 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
1370                         WREG32(mmGB_TILE_MODE0 + reg_offset, tile[reg_offset]);
1371                 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
1372                         if (reg_offset != 7)
1373                                 WREG32(mmGB_MACROTILE_MODE0 + reg_offset, macrotile[reg_offset]);
1374                 break;
1375         case CHIP_KABINI:
1376         case CHIP_KAVERI:
1377         case CHIP_MULLINS:
1378         default:
1379                 tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1380                            PIPE_CONFIG(ADDR_SURF_P2) |
1381                            TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
1382                            MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
1383                 tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1384                            PIPE_CONFIG(ADDR_SURF_P2) |
1385                            TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
1386                            MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
1387                 tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1388                            PIPE_CONFIG(ADDR_SURF_P2) |
1389                            TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
1390                            MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
1391                 tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1392                            PIPE_CONFIG(ADDR_SURF_P2) |
1393                            TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
1394                            MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
1395                 tile[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1396                            PIPE_CONFIG(ADDR_SURF_P2) |
1397                            MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1398                            TILE_SPLIT(split_equal_to_row_size));
1399                 tile[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1400                            PIPE_CONFIG(ADDR_SURF_P2) |
1401                            MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
1402                 tile[6] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
1403                            PIPE_CONFIG(ADDR_SURF_P2) |
1404                            MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1405                            TILE_SPLIT(split_equal_to_row_size));
1406                 tile[7] = (TILE_SPLIT(split_equal_to_row_size));
1407                 tile[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
1408                            PIPE_CONFIG(ADDR_SURF_P2));
1409                 tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1410                            PIPE_CONFIG(ADDR_SURF_P2) |
1411                            MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
1412                 tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1413                             PIPE_CONFIG(ADDR_SURF_P2) |
1414                             MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
1415                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1416                 tile[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
1417                             PIPE_CONFIG(ADDR_SURF_P2) |
1418                             MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
1419                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
1420                 tile[12] = (TILE_SPLIT(split_equal_to_row_size));
1421                 tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1422                             PIPE_CONFIG(ADDR_SURF_P2) |
1423                             MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
1424                 tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1425                             PIPE_CONFIG(ADDR_SURF_P2) |
1426                             MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1427                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1428                 tile[15] = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) |
1429                             PIPE_CONFIG(ADDR_SURF_P2) |
1430                             MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1431                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1432                 tile[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
1433                             PIPE_CONFIG(ADDR_SURF_P2) |
1434                             MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1435                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
1436                 tile[17] = (TILE_SPLIT(split_equal_to_row_size));
1437                 tile[18] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
1438                             PIPE_CONFIG(ADDR_SURF_P2) |
1439                             MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1440                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
1441                 tile[19] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
1442                             PIPE_CONFIG(ADDR_SURF_P2) |
1443                             MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING));
1444                 tile[20] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
1445                             PIPE_CONFIG(ADDR_SURF_P2) |
1446                             MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
1447                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
1448                 tile[21] = (ARRAY_MODE(ARRAY_3D_TILED_THICK) |
1449                             PIPE_CONFIG(ADDR_SURF_P2) |
1450                             MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
1451                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
1452                 tile[22] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |
1453                             PIPE_CONFIG(ADDR_SURF_P2) |
1454                             MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
1455                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
1456                 tile[23] = (TILE_SPLIT(split_equal_to_row_size));
1457                 tile[24] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
1458                             PIPE_CONFIG(ADDR_SURF_P2) |
1459                             MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1460                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
1461                 tile[25] = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) |
1462                             PIPE_CONFIG(ADDR_SURF_P2) |
1463                             MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
1464                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
1465                 tile[26] = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) |
1466                             PIPE_CONFIG(ADDR_SURF_P2) |
1467                             MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
1468                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
1469                 tile[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1470                             PIPE_CONFIG(ADDR_SURF_P2) |
1471                             MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
1472                 tile[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1473                             PIPE_CONFIG(ADDR_SURF_P2) |
1474                             MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
1475                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1476                 tile[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
1477                             PIPE_CONFIG(ADDR_SURF_P2) |
1478                             MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
1479                             SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
1480                 tile[30] = (TILE_SPLIT(split_equal_to_row_size));
1481
1482                 macrotile[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1483                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1484                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1485                                 NUM_BANKS(ADDR_SURF_8_BANK));
1486                 macrotile[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1487                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1488                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1489                                 NUM_BANKS(ADDR_SURF_8_BANK));
1490                 macrotile[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1491                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1492                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1493                                 NUM_BANKS(ADDR_SURF_8_BANK));
1494                 macrotile[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1495                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1496                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1497                                 NUM_BANKS(ADDR_SURF_8_BANK));
1498                 macrotile[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1499                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1500                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1501                                 NUM_BANKS(ADDR_SURF_8_BANK));
1502                 macrotile[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1503                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1504                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1505                                 NUM_BANKS(ADDR_SURF_8_BANK));
1506                 macrotile[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1507                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1508                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1509                                 NUM_BANKS(ADDR_SURF_8_BANK));
1510                 macrotile[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
1511                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
1512                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1513                                 NUM_BANKS(ADDR_SURF_16_BANK));
1514                 macrotile[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
1515                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1516                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1517                                 NUM_BANKS(ADDR_SURF_16_BANK));
1518                 macrotile[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
1519                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1520                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1521                                 NUM_BANKS(ADDR_SURF_16_BANK));
1522                 macrotile[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
1523                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1524                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1525                                 NUM_BANKS(ADDR_SURF_16_BANK));
1526                 macrotile[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1527                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1528                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1529                                 NUM_BANKS(ADDR_SURF_16_BANK));
1530                 macrotile[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1531                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1532                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1533                                 NUM_BANKS(ADDR_SURF_16_BANK));
1534                 macrotile[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1535                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1536                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1537                                 NUM_BANKS(ADDR_SURF_8_BANK));
1538
1539                 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
1540                         WREG32(mmGB_TILE_MODE0 + reg_offset, tile[reg_offset]);
1541                 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
1542                         if (reg_offset != 7)
1543                                 WREG32(mmGB_MACROTILE_MODE0 + reg_offset, macrotile[reg_offset]);
1544                 break;
1545         }
1546 }
1547
1548 /**
1549  * gfx_v7_0_select_se_sh - select which SE, SH to address
1550  *
1551  * @adev: amdgpu_device pointer
1552  * @se_num: shader engine to address
1553  * @sh_num: sh block to address
1554  *
1555  * Select which SE, SH combinations to address. Certain
1556  * registers are instanced per SE or SH.  0xffffffff means
1557  * broadcast to all SEs or SHs (CIK).
1558  */
1559 static void gfx_v7_0_select_se_sh(struct amdgpu_device *adev,
1560                                   u32 se_num, u32 sh_num, u32 instance)
1561 {
1562         u32 data;
1563
1564         if (instance == 0xffffffff)
1565                 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES, 1);
1566         else
1567                 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_INDEX, instance);
1568
1569         if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
1570                 data |= GRBM_GFX_INDEX__SH_BROADCAST_WRITES_MASK |
1571                         GRBM_GFX_INDEX__SE_BROADCAST_WRITES_MASK;
1572         else if (se_num == 0xffffffff)
1573                 data |= GRBM_GFX_INDEX__SE_BROADCAST_WRITES_MASK |
1574                         (sh_num << GRBM_GFX_INDEX__SH_INDEX__SHIFT);
1575         else if (sh_num == 0xffffffff)
1576                 data |= GRBM_GFX_INDEX__SH_BROADCAST_WRITES_MASK |
1577                         (se_num << GRBM_GFX_INDEX__SE_INDEX__SHIFT);
1578         else
1579                 data |= (sh_num << GRBM_GFX_INDEX__SH_INDEX__SHIFT) |
1580                         (se_num << GRBM_GFX_INDEX__SE_INDEX__SHIFT);
1581         WREG32(mmGRBM_GFX_INDEX, data);
1582 }
1583
1584 /**
1585  * gfx_v7_0_get_rb_active_bitmap - computes the mask of enabled RBs
1586  *
1587  * @adev: amdgpu_device pointer
1588  *
1589  * Calculates the bitmask of enabled RBs (CIK).
1590  * Returns the enabled RB bitmask.
1591  */
1592 static u32 gfx_v7_0_get_rb_active_bitmap(struct amdgpu_device *adev)
1593 {
1594         u32 data, mask;
1595
1596         data = RREG32(mmCC_RB_BACKEND_DISABLE);
1597         data |= RREG32(mmGC_USER_RB_BACKEND_DISABLE);
1598
1599         data &= CC_RB_BACKEND_DISABLE__BACKEND_DISABLE_MASK;
1600         data >>= GC_USER_RB_BACKEND_DISABLE__BACKEND_DISABLE__SHIFT;
1601
1602         mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_backends_per_se /
1603                                          adev->gfx.config.max_sh_per_se);
1604
1605         return (~data) & mask;
1606 }
1607
1608 static void
1609 gfx_v7_0_raster_config(struct amdgpu_device *adev, u32 *rconf, u32 *rconf1)
1610 {
1611         switch (adev->asic_type) {
1612         case CHIP_BONAIRE:
1613                 *rconf |= RB_MAP_PKR0(2) | RB_XSEL2(1) | SE_MAP(2) |
1614                           SE_XSEL(1) | SE_YSEL(1);
1615                 *rconf1 |= 0x0;
1616                 break;
1617         case CHIP_HAWAII:
1618                 *rconf |= RB_MAP_PKR0(2) | RB_MAP_PKR1(2) |
1619                           RB_XSEL2(1) | PKR_MAP(2) | PKR_XSEL(1) |
1620                           PKR_YSEL(1) | SE_MAP(2) | SE_XSEL(2) |
1621                           SE_YSEL(3);
1622                 *rconf1 |= SE_PAIR_MAP(2) | SE_PAIR_XSEL(3) |
1623                            SE_PAIR_YSEL(2);
1624                 break;
1625         case CHIP_KAVERI:
1626                 *rconf |= RB_MAP_PKR0(2);
1627                 *rconf1 |= 0x0;
1628                 break;
1629         case CHIP_KABINI:
1630         case CHIP_MULLINS:
1631                 *rconf |= 0x0;
1632                 *rconf1 |= 0x0;
1633                 break;
1634         default:
1635                 DRM_ERROR("unknown asic: 0x%x\n", adev->asic_type);
1636                 break;
1637         }
1638 }
1639
1640 static void
1641 gfx_v7_0_write_harvested_raster_configs(struct amdgpu_device *adev,
1642                                         u32 raster_config, u32 raster_config_1,
1643                                         unsigned rb_mask, unsigned num_rb)
1644 {
1645         unsigned sh_per_se = max_t(unsigned, adev->gfx.config.max_sh_per_se, 1);
1646         unsigned num_se = max_t(unsigned, adev->gfx.config.max_shader_engines, 1);
1647         unsigned rb_per_pkr = min_t(unsigned, num_rb / num_se / sh_per_se, 2);
1648         unsigned rb_per_se = num_rb / num_se;
1649         unsigned se_mask[4];
1650         unsigned se;
1651
1652         se_mask[0] = ((1 << rb_per_se) - 1) & rb_mask;
1653         se_mask[1] = (se_mask[0] << rb_per_se) & rb_mask;
1654         se_mask[2] = (se_mask[1] << rb_per_se) & rb_mask;
1655         se_mask[3] = (se_mask[2] << rb_per_se) & rb_mask;
1656
1657         WARN_ON(!(num_se == 1 || num_se == 2 || num_se == 4));
1658         WARN_ON(!(sh_per_se == 1 || sh_per_se == 2));
1659         WARN_ON(!(rb_per_pkr == 1 || rb_per_pkr == 2));
1660
1661         if ((num_se > 2) && ((!se_mask[0] && !se_mask[1]) ||
1662                              (!se_mask[2] && !se_mask[3]))) {
1663                 raster_config_1 &= ~SE_PAIR_MAP_MASK;
1664
1665                 if (!se_mask[0] && !se_mask[1]) {
1666                         raster_config_1 |=
1667                                 SE_PAIR_MAP(RASTER_CONFIG_SE_PAIR_MAP_3);
1668                 } else {
1669                         raster_config_1 |=
1670                                 SE_PAIR_MAP(RASTER_CONFIG_SE_PAIR_MAP_0);
1671                 }
1672         }
1673
1674         for (se = 0; se < num_se; se++) {
1675                 unsigned raster_config_se = raster_config;
1676                 unsigned pkr0_mask = ((1 << rb_per_pkr) - 1) << (se * rb_per_se);
1677                 unsigned pkr1_mask = pkr0_mask << rb_per_pkr;
1678                 int idx = (se / 2) * 2;
1679
1680                 if ((num_se > 1) && (!se_mask[idx] || !se_mask[idx + 1])) {
1681                         raster_config_se &= ~SE_MAP_MASK;
1682
1683                         if (!se_mask[idx]) {
1684                                 raster_config_se |= SE_MAP(RASTER_CONFIG_SE_MAP_3);
1685                         } else {
1686                                 raster_config_se |= SE_MAP(RASTER_CONFIG_SE_MAP_0);
1687                         }
1688                 }
1689
1690                 pkr0_mask &= rb_mask;
1691                 pkr1_mask &= rb_mask;
1692                 if (rb_per_se > 2 && (!pkr0_mask || !pkr1_mask)) {
1693                         raster_config_se &= ~PKR_MAP_MASK;
1694
1695                         if (!pkr0_mask) {
1696                                 raster_config_se |= PKR_MAP(RASTER_CONFIG_PKR_MAP_3);
1697                         } else {
1698                                 raster_config_se |= PKR_MAP(RASTER_CONFIG_PKR_MAP_0);
1699                         }
1700                 }
1701
1702                 if (rb_per_se >= 2) {
1703                         unsigned rb0_mask = 1 << (se * rb_per_se);
1704                         unsigned rb1_mask = rb0_mask << 1;
1705
1706                         rb0_mask &= rb_mask;
1707                         rb1_mask &= rb_mask;
1708                         if (!rb0_mask || !rb1_mask) {
1709                                 raster_config_se &= ~RB_MAP_PKR0_MASK;
1710
1711                                 if (!rb0_mask) {
1712                                         raster_config_se |=
1713                                                 RB_MAP_PKR0(RASTER_CONFIG_RB_MAP_3);
1714                                 } else {
1715                                         raster_config_se |=
1716                                                 RB_MAP_PKR0(RASTER_CONFIG_RB_MAP_0);
1717                                 }
1718                         }
1719
1720                         if (rb_per_se > 2) {
1721                                 rb0_mask = 1 << (se * rb_per_se + rb_per_pkr);
1722                                 rb1_mask = rb0_mask << 1;
1723                                 rb0_mask &= rb_mask;
1724                                 rb1_mask &= rb_mask;
1725                                 if (!rb0_mask || !rb1_mask) {
1726                                         raster_config_se &= ~RB_MAP_PKR1_MASK;
1727
1728                                         if (!rb0_mask) {
1729                                                 raster_config_se |=
1730                                                         RB_MAP_PKR1(RASTER_CONFIG_RB_MAP_3);
1731                                         } else {
1732                                                 raster_config_se |=
1733                                                         RB_MAP_PKR1(RASTER_CONFIG_RB_MAP_0);
1734                                         }
1735                                 }
1736                         }
1737                 }
1738
1739                 /* GRBM_GFX_INDEX has a different offset on CI+ */
1740                 gfx_v7_0_select_se_sh(adev, se, 0xffffffff, 0xffffffff);
1741                 WREG32(mmPA_SC_RASTER_CONFIG, raster_config_se);
1742                 WREG32(mmPA_SC_RASTER_CONFIG_1, raster_config_1);
1743         }
1744
1745         /* GRBM_GFX_INDEX has a different offset on CI+ */
1746         gfx_v7_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
1747 }
1748
1749 /**
1750  * gfx_v7_0_setup_rb - setup the RBs on the asic
1751  *
1752  * @adev: amdgpu_device pointer
1753  * @se_num: number of SEs (shader engines) for the asic
1754  * @sh_per_se: number of SH blocks per SE for the asic
1755  *
1756  * Configures per-SE/SH RB registers (CIK).
1757  */
1758 static void gfx_v7_0_setup_rb(struct amdgpu_device *adev)
1759 {
1760         int i, j;
1761         u32 data;
1762         u32 raster_config = 0, raster_config_1 = 0;
1763         u32 active_rbs = 0;
1764         u32 rb_bitmap_width_per_sh = adev->gfx.config.max_backends_per_se /
1765                                         adev->gfx.config.max_sh_per_se;
1766         unsigned num_rb_pipes;
1767
1768         mutex_lock(&adev->grbm_idx_mutex);
1769         for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
1770                 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
1771                         gfx_v7_0_select_se_sh(adev, i, j, 0xffffffff);
1772                         data = gfx_v7_0_get_rb_active_bitmap(adev);
1773                         active_rbs |= data << ((i * adev->gfx.config.max_sh_per_se + j) *
1774                                                rb_bitmap_width_per_sh);
1775                 }
1776         }
1777         gfx_v7_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
1778
1779         adev->gfx.config.backend_enable_mask = active_rbs;
1780         adev->gfx.config.num_rbs = hweight32(active_rbs);
1781
1782         num_rb_pipes = min_t(unsigned, adev->gfx.config.max_backends_per_se *
1783                              adev->gfx.config.max_shader_engines, 16);
1784
1785         gfx_v7_0_raster_config(adev, &raster_config, &raster_config_1);
1786
1787         if (!adev->gfx.config.backend_enable_mask ||
1788                         adev->gfx.config.num_rbs >= num_rb_pipes) {
1789                 WREG32(mmPA_SC_RASTER_CONFIG, raster_config);
1790                 WREG32(mmPA_SC_RASTER_CONFIG_1, raster_config_1);
1791         } else {
1792                 gfx_v7_0_write_harvested_raster_configs(adev, raster_config, raster_config_1,
1793                                                         adev->gfx.config.backend_enable_mask,
1794                                                         num_rb_pipes);
1795         }
1796
1797         /* cache the values for userspace */
1798         for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
1799                 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
1800                         gfx_v7_0_select_se_sh(adev, i, j, 0xffffffff);
1801                         adev->gfx.config.rb_config[i][j].rb_backend_disable =
1802                                 RREG32(mmCC_RB_BACKEND_DISABLE);
1803                         adev->gfx.config.rb_config[i][j].user_rb_backend_disable =
1804                                 RREG32(mmGC_USER_RB_BACKEND_DISABLE);
1805                         adev->gfx.config.rb_config[i][j].raster_config =
1806                                 RREG32(mmPA_SC_RASTER_CONFIG);
1807                         adev->gfx.config.rb_config[i][j].raster_config_1 =
1808                                 RREG32(mmPA_SC_RASTER_CONFIG_1);
1809                 }
1810         }
1811         gfx_v7_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
1812         mutex_unlock(&adev->grbm_idx_mutex);
1813 }
1814
1815 /**
1816  * gfx_v7_0_init_compute_vmid - gart enable
1817  *
1818  * @adev: amdgpu_device pointer
1819  *
1820  * Initialize compute vmid sh_mem registers
1821  *
1822  */
1823 #define DEFAULT_SH_MEM_BASES    (0x6000)
1824 #define FIRST_COMPUTE_VMID      (8)
1825 #define LAST_COMPUTE_VMID       (16)
1826 static void gfx_v7_0_init_compute_vmid(struct amdgpu_device *adev)
1827 {
1828         int i;
1829         uint32_t sh_mem_config;
1830         uint32_t sh_mem_bases;
1831
1832         /*
1833          * Configure apertures:
1834          * LDS:         0x60000000'00000000 - 0x60000001'00000000 (4GB)
1835          * Scratch:     0x60000001'00000000 - 0x60000002'00000000 (4GB)
1836          * GPUVM:       0x60010000'00000000 - 0x60020000'00000000 (1TB)
1837         */
1838         sh_mem_bases = DEFAULT_SH_MEM_BASES | (DEFAULT_SH_MEM_BASES << 16);
1839         sh_mem_config = SH_MEM_ALIGNMENT_MODE_UNALIGNED <<
1840                         SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT;
1841         sh_mem_config |= MTYPE_NONCACHED << SH_MEM_CONFIG__DEFAULT_MTYPE__SHIFT;
1842         mutex_lock(&adev->srbm_mutex);
1843         for (i = FIRST_COMPUTE_VMID; i < LAST_COMPUTE_VMID; i++) {
1844                 cik_srbm_select(adev, 0, 0, 0, i);
1845                 /* CP and shaders */
1846                 WREG32(mmSH_MEM_CONFIG, sh_mem_config);
1847                 WREG32(mmSH_MEM_APE1_BASE, 1);
1848                 WREG32(mmSH_MEM_APE1_LIMIT, 0);
1849                 WREG32(mmSH_MEM_BASES, sh_mem_bases);
1850         }
1851         cik_srbm_select(adev, 0, 0, 0, 0);
1852         mutex_unlock(&adev->srbm_mutex);
1853
1854         /* Initialize all compute VMIDs to have no GDS, GWS, or OA
1855            acccess. These should be enabled by FW for target VMIDs. */
1856         for (i = FIRST_COMPUTE_VMID; i < LAST_COMPUTE_VMID; i++) {
1857                 WREG32(amdgpu_gds_reg_offset[i].mem_base, 0);
1858                 WREG32(amdgpu_gds_reg_offset[i].mem_size, 0);
1859                 WREG32(amdgpu_gds_reg_offset[i].gws, 0);
1860                 WREG32(amdgpu_gds_reg_offset[i].oa, 0);
1861         }
1862 }
1863
1864 static void gfx_v7_0_config_init(struct amdgpu_device *adev)
1865 {
1866         adev->gfx.config.double_offchip_lds_buf = 1;
1867 }
1868
1869 /**
1870  * gfx_v7_0_constants_init - setup the 3D engine
1871  *
1872  * @adev: amdgpu_device pointer
1873  *
1874  * init the gfx constants such as the 3D engine, tiling configuration
1875  * registers, maximum number of quad pipes, render backends...
1876  */
1877 static void gfx_v7_0_constants_init(struct amdgpu_device *adev)
1878 {
1879         u32 sh_mem_cfg, sh_static_mem_cfg, sh_mem_base;
1880         u32 tmp;
1881         int i;
1882
1883         WREG32(mmGRBM_CNTL, (0xff << GRBM_CNTL__READ_TIMEOUT__SHIFT));
1884
1885         WREG32(mmGB_ADDR_CONFIG, adev->gfx.config.gb_addr_config);
1886         WREG32(mmHDP_ADDR_CONFIG, adev->gfx.config.gb_addr_config);
1887         WREG32(mmDMIF_ADDR_CALC, adev->gfx.config.gb_addr_config);
1888
1889         gfx_v7_0_tiling_mode_table_init(adev);
1890
1891         gfx_v7_0_setup_rb(adev);
1892         gfx_v7_0_get_cu_info(adev);
1893         gfx_v7_0_config_init(adev);
1894
1895         /* set HW defaults for 3D engine */
1896         WREG32(mmCP_MEQ_THRESHOLDS,
1897                (0x30 << CP_MEQ_THRESHOLDS__MEQ1_START__SHIFT) |
1898                (0x60 << CP_MEQ_THRESHOLDS__MEQ2_START__SHIFT));
1899
1900         mutex_lock(&adev->grbm_idx_mutex);
1901         /*
1902          * making sure that the following register writes will be broadcasted
1903          * to all the shaders
1904          */
1905         gfx_v7_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
1906
1907         /* XXX SH_MEM regs */
1908         /* where to put LDS, scratch, GPUVM in FSA64 space */
1909         sh_mem_cfg = REG_SET_FIELD(0, SH_MEM_CONFIG, ALIGNMENT_MODE,
1910                                    SH_MEM_ALIGNMENT_MODE_UNALIGNED);
1911         sh_mem_cfg = REG_SET_FIELD(sh_mem_cfg, SH_MEM_CONFIG, DEFAULT_MTYPE,
1912                                    MTYPE_NC);
1913         sh_mem_cfg = REG_SET_FIELD(sh_mem_cfg, SH_MEM_CONFIG, APE1_MTYPE,
1914                                    MTYPE_UC);
1915         sh_mem_cfg = REG_SET_FIELD(sh_mem_cfg, SH_MEM_CONFIG, PRIVATE_ATC, 0);
1916
1917         sh_static_mem_cfg = REG_SET_FIELD(0, SH_STATIC_MEM_CONFIG,
1918                                    SWIZZLE_ENABLE, 1);
1919         sh_static_mem_cfg = REG_SET_FIELD(sh_static_mem_cfg, SH_STATIC_MEM_CONFIG,
1920                                    ELEMENT_SIZE, 1);
1921         sh_static_mem_cfg = REG_SET_FIELD(sh_static_mem_cfg, SH_STATIC_MEM_CONFIG,
1922                                    INDEX_STRIDE, 3);
1923         WREG32(mmSH_STATIC_MEM_CONFIG, sh_static_mem_cfg);
1924
1925         mutex_lock(&adev->srbm_mutex);
1926         for (i = 0; i < adev->vm_manager.id_mgr[0].num_ids; i++) {
1927                 if (i == 0)
1928                         sh_mem_base = 0;
1929                 else
1930                         sh_mem_base = adev->gmc.shared_aperture_start >> 48;
1931                 cik_srbm_select(adev, 0, 0, 0, i);
1932                 /* CP and shaders */
1933                 WREG32(mmSH_MEM_CONFIG, sh_mem_cfg);
1934                 WREG32(mmSH_MEM_APE1_BASE, 1);
1935                 WREG32(mmSH_MEM_APE1_LIMIT, 0);
1936                 WREG32(mmSH_MEM_BASES, sh_mem_base);
1937         }
1938         cik_srbm_select(adev, 0, 0, 0, 0);
1939         mutex_unlock(&adev->srbm_mutex);
1940
1941         gfx_v7_0_init_compute_vmid(adev);
1942
1943         WREG32(mmSX_DEBUG_1, 0x20);
1944
1945         WREG32(mmTA_CNTL_AUX, 0x00010000);
1946
1947         tmp = RREG32(mmSPI_CONFIG_CNTL);
1948         tmp |= 0x03000000;
1949         WREG32(mmSPI_CONFIG_CNTL, tmp);
1950
1951         WREG32(mmSQ_CONFIG, 1);
1952
1953         WREG32(mmDB_DEBUG, 0);
1954
1955         tmp = RREG32(mmDB_DEBUG2) & ~0xf00fffff;
1956         tmp |= 0x00000400;
1957         WREG32(mmDB_DEBUG2, tmp);
1958
1959         tmp = RREG32(mmDB_DEBUG3) & ~0x0002021c;
1960         tmp |= 0x00020200;
1961         WREG32(mmDB_DEBUG3, tmp);
1962
1963         tmp = RREG32(mmCB_HW_CONTROL) & ~0x00010000;
1964         tmp |= 0x00018208;
1965         WREG32(mmCB_HW_CONTROL, tmp);
1966
1967         WREG32(mmSPI_CONFIG_CNTL_1, (4 << SPI_CONFIG_CNTL_1__VTX_DONE_DELAY__SHIFT));
1968
1969         WREG32(mmPA_SC_FIFO_SIZE,
1970                 ((adev->gfx.config.sc_prim_fifo_size_frontend << PA_SC_FIFO_SIZE__SC_FRONTEND_PRIM_FIFO_SIZE__SHIFT) |
1971                 (adev->gfx.config.sc_prim_fifo_size_backend << PA_SC_FIFO_SIZE__SC_BACKEND_PRIM_FIFO_SIZE__SHIFT) |
1972                 (adev->gfx.config.sc_hiz_tile_fifo_size << PA_SC_FIFO_SIZE__SC_HIZ_TILE_FIFO_SIZE__SHIFT) |
1973                 (adev->gfx.config.sc_earlyz_tile_fifo_size << PA_SC_FIFO_SIZE__SC_EARLYZ_TILE_FIFO_SIZE__SHIFT)));
1974
1975         WREG32(mmVGT_NUM_INSTANCES, 1);
1976
1977         WREG32(mmCP_PERFMON_CNTL, 0);
1978
1979         WREG32(mmSQ_CONFIG, 0);
1980
1981         WREG32(mmPA_SC_FORCE_EOV_MAX_CNTS,
1982                 ((4095 << PA_SC_FORCE_EOV_MAX_CNTS__FORCE_EOV_MAX_CLK_CNT__SHIFT) |
1983                 (255 << PA_SC_FORCE_EOV_MAX_CNTS__FORCE_EOV_MAX_REZ_CNT__SHIFT)));
1984
1985         WREG32(mmVGT_CACHE_INVALIDATION,
1986                 (VC_AND_TC << VGT_CACHE_INVALIDATION__CACHE_INVALIDATION__SHIFT) |
1987                 (ES_AND_GS_AUTO << VGT_CACHE_INVALIDATION__AUTO_INVLD_EN__SHIFT));
1988
1989         WREG32(mmVGT_GS_VERTEX_REUSE, 16);
1990         WREG32(mmPA_SC_LINE_STIPPLE_STATE, 0);
1991
1992         WREG32(mmPA_CL_ENHANCE, PA_CL_ENHANCE__CLIP_VTX_REORDER_ENA_MASK |
1993                         (3 << PA_CL_ENHANCE__NUM_CLIP_SEQ__SHIFT));
1994         WREG32(mmPA_SC_ENHANCE, PA_SC_ENHANCE__ENABLE_PA_SC_OUT_OF_ORDER_MASK);
1995
1996         tmp = RREG32(mmSPI_ARB_PRIORITY);
1997         tmp = REG_SET_FIELD(tmp, SPI_ARB_PRIORITY, PIPE_ORDER_TS0, 2);
1998         tmp = REG_SET_FIELD(tmp, SPI_ARB_PRIORITY, PIPE_ORDER_TS1, 2);
1999         tmp = REG_SET_FIELD(tmp, SPI_ARB_PRIORITY, PIPE_ORDER_TS2, 2);
2000         tmp = REG_SET_FIELD(tmp, SPI_ARB_PRIORITY, PIPE_ORDER_TS3, 2);
2001         WREG32(mmSPI_ARB_PRIORITY, tmp);
2002
2003         mutex_unlock(&adev->grbm_idx_mutex);
2004
2005         udelay(50);
2006 }
2007
2008 /*
2009  * GPU scratch registers helpers function.
2010  */
2011 /**
2012  * gfx_v7_0_scratch_init - setup driver info for CP scratch regs
2013  *
2014  * @adev: amdgpu_device pointer
2015  *
2016  * Set up the number and offset of the CP scratch registers.
2017  * NOTE: use of CP scratch registers is a legacy inferface and
2018  * is not used by default on newer asics (r6xx+).  On newer asics,
2019  * memory buffers are used for fences rather than scratch regs.
2020  */
2021 static void gfx_v7_0_scratch_init(struct amdgpu_device *adev)
2022 {
2023         adev->gfx.scratch.num_reg = 8;
2024         adev->gfx.scratch.reg_base = mmSCRATCH_REG0;
2025         adev->gfx.scratch.free_mask = (1u << adev->gfx.scratch.num_reg) - 1;
2026 }
2027
2028 /**
2029  * gfx_v7_0_ring_test_ring - basic gfx ring test
2030  *
2031  * @adev: amdgpu_device pointer
2032  * @ring: amdgpu_ring structure holding ring information
2033  *
2034  * Allocate a scratch register and write to it using the gfx ring (CIK).
2035  * Provides a basic gfx ring test to verify that the ring is working.
2036  * Used by gfx_v7_0_cp_gfx_resume();
2037  * Returns 0 on success, error on failure.
2038  */
2039 static int gfx_v7_0_ring_test_ring(struct amdgpu_ring *ring)
2040 {
2041         struct amdgpu_device *adev = ring->adev;
2042         uint32_t scratch;
2043         uint32_t tmp = 0;
2044         unsigned i;
2045         int r;
2046
2047         r = amdgpu_gfx_scratch_get(adev, &scratch);
2048         if (r)
2049                 return r;
2050
2051         WREG32(scratch, 0xCAFEDEAD);
2052         r = amdgpu_ring_alloc(ring, 3);
2053         if (r)
2054                 goto error_free_scratch;
2055
2056         amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
2057         amdgpu_ring_write(ring, (scratch - PACKET3_SET_UCONFIG_REG_START));
2058         amdgpu_ring_write(ring, 0xDEADBEEF);
2059         amdgpu_ring_commit(ring);
2060
2061         for (i = 0; i < adev->usec_timeout; i++) {
2062                 tmp = RREG32(scratch);
2063                 if (tmp == 0xDEADBEEF)
2064                         break;
2065                 udelay(1);
2066         }
2067         if (i >= adev->usec_timeout)
2068                 r = -ETIMEDOUT;
2069
2070 error_free_scratch:
2071         amdgpu_gfx_scratch_free(adev, scratch);
2072         return r;
2073 }
2074
2075 /**
2076  * gfx_v7_0_ring_emit_hdp - emit an hdp flush on the cp
2077  *
2078  * @adev: amdgpu_device pointer
2079  * @ridx: amdgpu ring index
2080  *
2081  * Emits an hdp flush on the cp.
2082  */
2083 static void gfx_v7_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)
2084 {
2085         u32 ref_and_mask;
2086         int usepfp = ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE ? 0 : 1;
2087
2088         if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) {
2089                 switch (ring->me) {
2090                 case 1:
2091                         ref_and_mask = GPU_HDP_FLUSH_DONE__CP2_MASK << ring->pipe;
2092                         break;
2093                 case 2:
2094                         ref_and_mask = GPU_HDP_FLUSH_DONE__CP6_MASK << ring->pipe;
2095                         break;
2096                 default:
2097                         return;
2098                 }
2099         } else {
2100                 ref_and_mask = GPU_HDP_FLUSH_DONE__CP0_MASK;
2101         }
2102
2103         amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
2104         amdgpu_ring_write(ring, (WAIT_REG_MEM_OPERATION(1) | /* write, wait, write */
2105                                  WAIT_REG_MEM_FUNCTION(3) |  /* == */
2106                                  WAIT_REG_MEM_ENGINE(usepfp)));   /* pfp or me */
2107         amdgpu_ring_write(ring, mmGPU_HDP_FLUSH_REQ);
2108         amdgpu_ring_write(ring, mmGPU_HDP_FLUSH_DONE);
2109         amdgpu_ring_write(ring, ref_and_mask);
2110         amdgpu_ring_write(ring, ref_and_mask);
2111         amdgpu_ring_write(ring, 0x20); /* poll interval */
2112 }
2113
2114 static void gfx_v7_0_ring_emit_vgt_flush(struct amdgpu_ring *ring)
2115 {
2116         amdgpu_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE, 0));
2117         amdgpu_ring_write(ring, EVENT_TYPE(VS_PARTIAL_FLUSH) |
2118                 EVENT_INDEX(4));
2119
2120         amdgpu_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE, 0));
2121         amdgpu_ring_write(ring, EVENT_TYPE(VGT_FLUSH) |
2122                 EVENT_INDEX(0));
2123 }
2124
2125 /**
2126  * gfx_v7_0_ring_emit_fence_gfx - emit a fence on the gfx ring
2127  *
2128  * @adev: amdgpu_device pointer
2129  * @fence: amdgpu fence object
2130  *
2131  * Emits a fence sequnce number on the gfx ring and flushes
2132  * GPU caches.
2133  */
2134 static void gfx_v7_0_ring_emit_fence_gfx(struct amdgpu_ring *ring, u64 addr,
2135                                          u64 seq, unsigned flags)
2136 {
2137         bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT;
2138         bool int_sel = flags & AMDGPU_FENCE_FLAG_INT;
2139         /* Workaround for cache flush problems. First send a dummy EOP
2140          * event down the pipe with seq one below.
2141          */
2142         amdgpu_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
2143         amdgpu_ring_write(ring, (EOP_TCL1_ACTION_EN |
2144                                  EOP_TC_ACTION_EN |
2145                                  EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
2146                                  EVENT_INDEX(5)));
2147         amdgpu_ring_write(ring, addr & 0xfffffffc);
2148         amdgpu_ring_write(ring, (upper_32_bits(addr) & 0xffff) |
2149                                 DATA_SEL(1) | INT_SEL(0));
2150         amdgpu_ring_write(ring, lower_32_bits(seq - 1));
2151         amdgpu_ring_write(ring, upper_32_bits(seq - 1));
2152
2153         /* Then send the real EOP event down the pipe. */
2154         amdgpu_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
2155         amdgpu_ring_write(ring, (EOP_TCL1_ACTION_EN |
2156                                  EOP_TC_ACTION_EN |
2157                                  EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
2158                                  EVENT_INDEX(5)));
2159         amdgpu_ring_write(ring, addr & 0xfffffffc);
2160         amdgpu_ring_write(ring, (upper_32_bits(addr) & 0xffff) |
2161                                 DATA_SEL(write64bit ? 2 : 1) | INT_SEL(int_sel ? 2 : 0));
2162         amdgpu_ring_write(ring, lower_32_bits(seq));
2163         amdgpu_ring_write(ring, upper_32_bits(seq));
2164 }
2165
2166 /**
2167  * gfx_v7_0_ring_emit_fence_compute - emit a fence on the compute ring
2168  *
2169  * @adev: amdgpu_device pointer
2170  * @fence: amdgpu fence object
2171  *
2172  * Emits a fence sequnce number on the compute ring and flushes
2173  * GPU caches.
2174  */
2175 static void gfx_v7_0_ring_emit_fence_compute(struct amdgpu_ring *ring,
2176                                              u64 addr, u64 seq,
2177                                              unsigned flags)
2178 {
2179         bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT;
2180         bool int_sel = flags & AMDGPU_FENCE_FLAG_INT;
2181
2182         /* RELEASE_MEM - flush caches, send int */
2183         amdgpu_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 5));
2184         amdgpu_ring_write(ring, (EOP_TCL1_ACTION_EN |
2185                                  EOP_TC_ACTION_EN |
2186                                  EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
2187                                  EVENT_INDEX(5)));
2188         amdgpu_ring_write(ring, DATA_SEL(write64bit ? 2 : 1) | INT_SEL(int_sel ? 2 : 0));
2189         amdgpu_ring_write(ring, addr & 0xfffffffc);
2190         amdgpu_ring_write(ring, upper_32_bits(addr));
2191         amdgpu_ring_write(ring, lower_32_bits(seq));
2192         amdgpu_ring_write(ring, upper_32_bits(seq));
2193 }
2194
2195 /*
2196  * IB stuff
2197  */
2198 /**
2199  * gfx_v7_0_ring_emit_ib - emit an IB (Indirect Buffer) on the ring
2200  *
2201  * @ring: amdgpu_ring structure holding ring information
2202  * @ib: amdgpu indirect buffer object
2203  *
2204  * Emits an DE (drawing engine) or CE (constant engine) IB
2205  * on the gfx ring.  IBs are usually generated by userspace
2206  * acceleration drivers and submitted to the kernel for
2207  * sheduling on the ring.  This function schedules the IB
2208  * on the gfx ring for execution by the GPU.
2209  */
2210 static void gfx_v7_0_ring_emit_ib_gfx(struct amdgpu_ring *ring,
2211                                         struct amdgpu_job *job,
2212                                         struct amdgpu_ib *ib,
2213                                         uint32_t flags)
2214 {
2215         unsigned vmid = AMDGPU_JOB_GET_VMID(job);
2216         u32 header, control = 0;
2217
2218         /* insert SWITCH_BUFFER packet before first IB in the ring frame */
2219         if (flags & AMDGPU_HAVE_CTX_SWITCH) {
2220                 amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
2221                 amdgpu_ring_write(ring, 0);
2222         }
2223
2224         if (ib->flags & AMDGPU_IB_FLAG_CE)
2225                 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
2226         else
2227                 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
2228
2229         control |= ib->length_dw | (vmid << 24);
2230
2231         amdgpu_ring_write(ring, header);
2232         amdgpu_ring_write(ring,
2233 #ifdef __BIG_ENDIAN
2234                           (2 << 0) |
2235 #endif
2236                           (ib->gpu_addr & 0xFFFFFFFC));
2237         amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
2238         amdgpu_ring_write(ring, control);
2239 }
2240
2241 static void gfx_v7_0_ring_emit_ib_compute(struct amdgpu_ring *ring,
2242                                           struct amdgpu_job *job,
2243                                           struct amdgpu_ib *ib,
2244                                           uint32_t flags)
2245 {
2246         unsigned vmid = AMDGPU_JOB_GET_VMID(job);
2247         u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vmid << 24);
2248
2249         /* Currently, there is a high possibility to get wave ID mismatch
2250          * between ME and GDS, leading to a hw deadlock, because ME generates
2251          * different wave IDs than the GDS expects. This situation happens
2252          * randomly when at least 5 compute pipes use GDS ordered append.
2253          * The wave IDs generated by ME are also wrong after suspend/resume.
2254          * Those are probably bugs somewhere else in the kernel driver.
2255          *
2256          * Writing GDS_COMPUTE_MAX_WAVE_ID resets wave ID counters in ME and
2257          * GDS to 0 for this ring (me/pipe).
2258          */
2259         if (ib->flags & AMDGPU_IB_FLAG_RESET_GDS_MAX_WAVE_ID) {
2260                 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2261                 amdgpu_ring_write(ring, mmGDS_COMPUTE_MAX_WAVE_ID - PACKET3_SET_CONFIG_REG_START);
2262                 amdgpu_ring_write(ring, ring->adev->gds.gds_compute_max_wave_id);
2263         }
2264
2265         amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
2266         amdgpu_ring_write(ring,
2267 #ifdef __BIG_ENDIAN
2268                                           (2 << 0) |
2269 #endif
2270                                           (ib->gpu_addr & 0xFFFFFFFC));
2271         amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
2272         amdgpu_ring_write(ring, control);
2273 }
2274
2275 static void gfx_v7_ring_emit_cntxcntl(struct amdgpu_ring *ring, uint32_t flags)
2276 {
2277         uint32_t dw2 = 0;
2278
2279         dw2 |= 0x80000000; /* set load_enable otherwise this package is just NOPs */
2280         if (flags & AMDGPU_HAVE_CTX_SWITCH) {
2281                 gfx_v7_0_ring_emit_vgt_flush(ring);
2282                 /* set load_global_config & load_global_uconfig */
2283                 dw2 |= 0x8001;
2284                 /* set load_cs_sh_regs */
2285                 dw2 |= 0x01000000;
2286                 /* set load_per_context_state & load_gfx_sh_regs */
2287                 dw2 |= 0x10002;
2288         }
2289
2290         amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
2291         amdgpu_ring_write(ring, dw2);
2292         amdgpu_ring_write(ring, 0);
2293 }
2294
2295 /**
2296  * gfx_v7_0_ring_test_ib - basic ring IB test
2297  *
2298  * @ring: amdgpu_ring structure holding ring information
2299  *
2300  * Allocate an IB and execute it on the gfx ring (CIK).
2301  * Provides a basic gfx ring test to verify that IBs are working.
2302  * Returns 0 on success, error on failure.
2303  */
2304 static int gfx_v7_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
2305 {
2306         struct amdgpu_device *adev = ring->adev;
2307         struct amdgpu_ib ib;
2308         struct dma_fence *f = NULL;
2309         uint32_t scratch;
2310         uint32_t tmp = 0;
2311         long r;
2312
2313         r = amdgpu_gfx_scratch_get(adev, &scratch);
2314         if (r)
2315                 return r;
2316
2317         WREG32(scratch, 0xCAFEDEAD);
2318         memset(&ib, 0, sizeof(ib));
2319         r = amdgpu_ib_get(adev, NULL, 256, &ib);
2320         if (r)
2321                 goto err1;
2322
2323         ib.ptr[0] = PACKET3(PACKET3_SET_UCONFIG_REG, 1);
2324         ib.ptr[1] = ((scratch - PACKET3_SET_UCONFIG_REG_START));
2325         ib.ptr[2] = 0xDEADBEEF;
2326         ib.length_dw = 3;
2327
2328         r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
2329         if (r)
2330                 goto err2;
2331
2332         r = dma_fence_wait_timeout(f, false, timeout);
2333         if (r == 0) {
2334                 r = -ETIMEDOUT;
2335                 goto err2;
2336         } else if (r < 0) {
2337                 goto err2;
2338         }
2339         tmp = RREG32(scratch);
2340         if (tmp == 0xDEADBEEF)
2341                 r = 0;
2342         else
2343                 r = -EINVAL;
2344
2345 err2:
2346         amdgpu_ib_free(adev, &ib, NULL);
2347         dma_fence_put(f);
2348 err1:
2349         amdgpu_gfx_scratch_free(adev, scratch);
2350         return r;
2351 }
2352
2353 /*
2354  * CP.
2355  * On CIK, gfx and compute now have independant command processors.
2356  *
2357  * GFX
2358  * Gfx consists of a single ring and can process both gfx jobs and
2359  * compute jobs.  The gfx CP consists of three microengines (ME):
2360  * PFP - Pre-Fetch Parser
2361  * ME - Micro Engine
2362  * CE - Constant Engine
2363  * The PFP and ME make up what is considered the Drawing Engine (DE).
2364  * The CE is an asynchronous engine used for updating buffer desciptors
2365  * used by the DE so that they can be loaded into cache in parallel
2366  * while the DE is processing state update packets.
2367  *
2368  * Compute
2369  * The compute CP consists of two microengines (ME):
2370  * MEC1 - Compute MicroEngine 1
2371  * MEC2 - Compute MicroEngine 2
2372  * Each MEC supports 4 compute pipes and each pipe supports 8 queues.
2373  * The queues are exposed to userspace and are programmed directly
2374  * by the compute runtime.
2375  */
2376 /**
2377  * gfx_v7_0_cp_gfx_enable - enable/disable the gfx CP MEs
2378  *
2379  * @adev: amdgpu_device pointer
2380  * @enable: enable or disable the MEs
2381  *
2382  * Halts or unhalts the gfx MEs.
2383  */
2384 static void gfx_v7_0_cp_gfx_enable(struct amdgpu_device *adev, bool enable)
2385 {
2386         int i;
2387
2388         if (enable) {
2389                 WREG32(mmCP_ME_CNTL, 0);
2390         } else {
2391                 WREG32(mmCP_ME_CNTL, (CP_ME_CNTL__ME_HALT_MASK | CP_ME_CNTL__PFP_HALT_MASK | CP_ME_CNTL__CE_HALT_MASK));
2392                 for (i = 0; i < adev->gfx.num_gfx_rings; i++)
2393                         adev->gfx.gfx_ring[i].sched.ready = false;
2394         }
2395         udelay(50);
2396 }
2397
2398 /**
2399  * gfx_v7_0_cp_gfx_load_microcode - load the gfx CP ME ucode
2400  *
2401  * @adev: amdgpu_device pointer
2402  *
2403  * Loads the gfx PFP, ME, and CE ucode.
2404  * Returns 0 for success, -EINVAL if the ucode is not available.
2405  */
2406 static int gfx_v7_0_cp_gfx_load_microcode(struct amdgpu_device *adev)
2407 {
2408         const struct gfx_firmware_header_v1_0 *pfp_hdr;
2409         const struct gfx_firmware_header_v1_0 *ce_hdr;
2410         const struct gfx_firmware_header_v1_0 *me_hdr;
2411         const __le32 *fw_data;
2412         unsigned i, fw_size;
2413
2414         if (!adev->gfx.me_fw || !adev->gfx.pfp_fw || !adev->gfx.ce_fw)
2415                 return -EINVAL;
2416
2417         pfp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.pfp_fw->data;
2418         ce_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.ce_fw->data;
2419         me_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.me_fw->data;
2420
2421         amdgpu_ucode_print_gfx_hdr(&pfp_hdr->header);
2422         amdgpu_ucode_print_gfx_hdr(&ce_hdr->header);
2423         amdgpu_ucode_print_gfx_hdr(&me_hdr->header);
2424         adev->gfx.pfp_fw_version = le32_to_cpu(pfp_hdr->header.ucode_version);
2425         adev->gfx.ce_fw_version = le32_to_cpu(ce_hdr->header.ucode_version);
2426         adev->gfx.me_fw_version = le32_to_cpu(me_hdr->header.ucode_version);
2427         adev->gfx.me_feature_version = le32_to_cpu(me_hdr->ucode_feature_version);
2428         adev->gfx.ce_feature_version = le32_to_cpu(ce_hdr->ucode_feature_version);
2429         adev->gfx.pfp_feature_version = le32_to_cpu(pfp_hdr->ucode_feature_version);
2430
2431         gfx_v7_0_cp_gfx_enable(adev, false);
2432
2433         /* PFP */
2434         fw_data = (const __le32 *)
2435                 (adev->gfx.pfp_fw->data +
2436                  le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes));
2437         fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4;
2438         WREG32(mmCP_PFP_UCODE_ADDR, 0);
2439         for (i = 0; i < fw_size; i++)
2440                 WREG32(mmCP_PFP_UCODE_DATA, le32_to_cpup(fw_data++));
2441         WREG32(mmCP_PFP_UCODE_ADDR, adev->gfx.pfp_fw_version);
2442
2443         /* CE */
2444         fw_data = (const __le32 *)
2445                 (adev->gfx.ce_fw->data +
2446                  le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes));
2447         fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4;
2448         WREG32(mmCP_CE_UCODE_ADDR, 0);
2449         for (i = 0; i < fw_size; i++)
2450                 WREG32(mmCP_CE_UCODE_DATA, le32_to_cpup(fw_data++));
2451         WREG32(mmCP_CE_UCODE_ADDR, adev->gfx.ce_fw_version);
2452
2453         /* ME */
2454         fw_data = (const __le32 *)
2455                 (adev->gfx.me_fw->data +
2456                  le32_to_cpu(me_hdr->header.ucode_array_offset_bytes));
2457         fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4;
2458         WREG32(mmCP_ME_RAM_WADDR, 0);
2459         for (i = 0; i < fw_size; i++)
2460                 WREG32(mmCP_ME_RAM_DATA, le32_to_cpup(fw_data++));
2461         WREG32(mmCP_ME_RAM_WADDR, adev->gfx.me_fw_version);
2462
2463         return 0;
2464 }
2465
2466 /**
2467  * gfx_v7_0_cp_gfx_start - start the gfx ring
2468  *
2469  * @adev: amdgpu_device pointer
2470  *
2471  * Enables the ring and loads the clear state context and other
2472  * packets required to init the ring.
2473  * Returns 0 for success, error for failure.
2474  */
2475 static int gfx_v7_0_cp_gfx_start(struct amdgpu_device *adev)
2476 {
2477         struct amdgpu_ring *ring = &adev->gfx.gfx_ring[0];
2478         const struct cs_section_def *sect = NULL;
2479         const struct cs_extent_def *ext = NULL;
2480         int r, i;
2481
2482         /* init the CP */
2483         WREG32(mmCP_MAX_CONTEXT, adev->gfx.config.max_hw_contexts - 1);
2484         WREG32(mmCP_ENDIAN_SWAP, 0);
2485         WREG32(mmCP_DEVICE_ID, 1);
2486
2487         gfx_v7_0_cp_gfx_enable(adev, true);
2488
2489         r = amdgpu_ring_alloc(ring, gfx_v7_0_get_csb_size(adev) + 8);
2490         if (r) {
2491                 DRM_ERROR("amdgpu: cp failed to lock ring (%d).\n", r);
2492                 return r;
2493         }
2494
2495         /* init the CE partitions.  CE only used for gfx on CIK */
2496         amdgpu_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
2497         amdgpu_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
2498         amdgpu_ring_write(ring, 0x8000);
2499         amdgpu_ring_write(ring, 0x8000);
2500
2501         /* clear state buffer */
2502         amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2503         amdgpu_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
2504
2505         amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
2506         amdgpu_ring_write(ring, 0x80000000);
2507         amdgpu_ring_write(ring, 0x80000000);
2508
2509         for (sect = adev->gfx.rlc.cs_data; sect->section != NULL; ++sect) {
2510                 for (ext = sect->section; ext->extent != NULL; ++ext) {
2511                         if (sect->id == SECT_CONTEXT) {
2512                                 amdgpu_ring_write(ring,
2513                                                   PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
2514                                 amdgpu_ring_write(ring, ext->reg_index - PACKET3_SET_CONTEXT_REG_START);
2515                                 for (i = 0; i < ext->reg_count; i++)
2516                                         amdgpu_ring_write(ring, ext->extent[i]);
2517                         }
2518                 }
2519         }
2520
2521         amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
2522         amdgpu_ring_write(ring, mmPA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
2523         amdgpu_ring_write(ring, adev->gfx.config.rb_config[0][0].raster_config);
2524         amdgpu_ring_write(ring, adev->gfx.config.rb_config[0][0].raster_config_1);
2525
2526         amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2527         amdgpu_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
2528
2529         amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
2530         amdgpu_ring_write(ring, 0);
2531
2532         amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
2533         amdgpu_ring_write(ring, 0x00000316);
2534         amdgpu_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
2535         amdgpu_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
2536
2537         amdgpu_ring_commit(ring);
2538
2539         return 0;
2540 }
2541
2542 /**
2543  * gfx_v7_0_cp_gfx_resume - setup the gfx ring buffer registers
2544  *
2545  * @adev: amdgpu_device pointer
2546  *
2547  * Program the location and size of the gfx ring buffer
2548  * and test it to make sure it's working.
2549  * Returns 0 for success, error for failure.
2550  */
2551 static int gfx_v7_0_cp_gfx_resume(struct amdgpu_device *adev)
2552 {
2553         struct amdgpu_ring *ring;
2554         u32 tmp;
2555         u32 rb_bufsz;
2556         u64 rb_addr, rptr_addr;
2557         int r;
2558
2559         WREG32(mmCP_SEM_WAIT_TIMER, 0x0);
2560         if (adev->asic_type != CHIP_HAWAII)
2561                 WREG32(mmCP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
2562
2563         /* Set the write pointer delay */
2564         WREG32(mmCP_RB_WPTR_DELAY, 0);
2565
2566         /* set the RB to use vmid 0 */
2567         WREG32(mmCP_RB_VMID, 0);
2568
2569         WREG32(mmSCRATCH_ADDR, 0);
2570
2571         /* ring 0 - compute and gfx */
2572         /* Set ring buffer size */
2573         ring = &adev->gfx.gfx_ring[0];
2574         rb_bufsz = order_base_2(ring->ring_size / 8);
2575         tmp = (order_base_2(AMDGPU_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
2576 #ifdef __BIG_ENDIAN
2577         tmp |= 2 << CP_RB0_CNTL__BUF_SWAP__SHIFT;
2578 #endif
2579         WREG32(mmCP_RB0_CNTL, tmp);
2580
2581         /* Initialize the ring buffer's read and write pointers */
2582         WREG32(mmCP_RB0_CNTL, tmp | CP_RB0_CNTL__RB_RPTR_WR_ENA_MASK);
2583         ring->wptr = 0;
2584         WREG32(mmCP_RB0_WPTR, lower_32_bits(ring->wptr));
2585
2586         /* set the wb address wether it's enabled or not */
2587         rptr_addr = adev->wb.gpu_addr + (ring->rptr_offs * 4);
2588         WREG32(mmCP_RB0_RPTR_ADDR, lower_32_bits(rptr_addr));
2589         WREG32(mmCP_RB0_RPTR_ADDR_HI, upper_32_bits(rptr_addr) & 0xFF);
2590
2591         /* scratch register shadowing is no longer supported */
2592         WREG32(mmSCRATCH_UMSK, 0);
2593
2594         mdelay(1);
2595         WREG32(mmCP_RB0_CNTL, tmp);
2596
2597         rb_addr = ring->gpu_addr >> 8;
2598         WREG32(mmCP_RB0_BASE, rb_addr);
2599         WREG32(mmCP_RB0_BASE_HI, upper_32_bits(rb_addr));
2600
2601         /* start the ring */
2602         gfx_v7_0_cp_gfx_start(adev);
2603         r = amdgpu_ring_test_helper(ring);
2604         if (r)
2605                 return r;
2606
2607         return 0;
2608 }
2609
2610 static u64 gfx_v7_0_ring_get_rptr(struct amdgpu_ring *ring)
2611 {
2612         return ring->adev->wb.wb[ring->rptr_offs];
2613 }
2614
2615 static u64 gfx_v7_0_ring_get_wptr_gfx(struct amdgpu_ring *ring)
2616 {
2617         struct amdgpu_device *adev = ring->adev;
2618
2619         return RREG32(mmCP_RB0_WPTR);
2620 }
2621
2622 static void gfx_v7_0_ring_set_wptr_gfx(struct amdgpu_ring *ring)
2623 {
2624         struct amdgpu_device *adev = ring->adev;
2625
2626         WREG32(mmCP_RB0_WPTR, lower_32_bits(ring->wptr));
2627         (void)RREG32(mmCP_RB0_WPTR);
2628 }
2629
2630 static u64 gfx_v7_0_ring_get_wptr_compute(struct amdgpu_ring *ring)
2631 {
2632         /* XXX check if swapping is necessary on BE */
2633         return ring->adev->wb.wb[ring->wptr_offs];
2634 }
2635
2636 static void gfx_v7_0_ring_set_wptr_compute(struct amdgpu_ring *ring)
2637 {
2638         struct amdgpu_device *adev = ring->adev;
2639
2640         /* XXX check if swapping is necessary on BE */
2641         adev->wb.wb[ring->wptr_offs] = lower_32_bits(ring->wptr);
2642         WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));
2643 }
2644
2645 /**
2646  * gfx_v7_0_cp_compute_enable - enable/disable the compute CP MEs
2647  *
2648  * @adev: amdgpu_device pointer
2649  * @enable: enable or disable the MEs
2650  *
2651  * Halts or unhalts the compute MEs.
2652  */
2653 static void gfx_v7_0_cp_compute_enable(struct amdgpu_device *adev, bool enable)
2654 {
2655         int i;
2656
2657         if (enable) {
2658                 WREG32(mmCP_MEC_CNTL, 0);
2659         } else {
2660                 WREG32(mmCP_MEC_CNTL, (CP_MEC_CNTL__MEC_ME1_HALT_MASK | CP_MEC_CNTL__MEC_ME2_HALT_MASK));
2661                 for (i = 0; i < adev->gfx.num_compute_rings; i++)
2662                         adev->gfx.compute_ring[i].sched.ready = false;
2663         }
2664         udelay(50);
2665 }
2666
2667 /**
2668  * gfx_v7_0_cp_compute_load_microcode - load the compute CP ME ucode
2669  *
2670  * @adev: amdgpu_device pointer
2671  *
2672  * Loads the compute MEC1&2 ucode.
2673  * Returns 0 for success, -EINVAL if the ucode is not available.
2674  */
2675 static int gfx_v7_0_cp_compute_load_microcode(struct amdgpu_device *adev)
2676 {
2677         const struct gfx_firmware_header_v1_0 *mec_hdr;
2678         const __le32 *fw_data;
2679         unsigned i, fw_size;
2680
2681         if (!adev->gfx.mec_fw)
2682                 return -EINVAL;
2683
2684         mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
2685         amdgpu_ucode_print_gfx_hdr(&mec_hdr->header);
2686         adev->gfx.mec_fw_version = le32_to_cpu(mec_hdr->header.ucode_version);
2687         adev->gfx.mec_feature_version = le32_to_cpu(
2688                                         mec_hdr->ucode_feature_version);
2689
2690         gfx_v7_0_cp_compute_enable(adev, false);
2691
2692         /* MEC1 */
2693         fw_data = (const __le32 *)
2694                 (adev->gfx.mec_fw->data +
2695                  le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));
2696         fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes) / 4;
2697         WREG32(mmCP_MEC_ME1_UCODE_ADDR, 0);
2698         for (i = 0; i < fw_size; i++)
2699                 WREG32(mmCP_MEC_ME1_UCODE_DATA, le32_to_cpup(fw_data++));
2700         WREG32(mmCP_MEC_ME1_UCODE_ADDR, 0);
2701
2702         if (adev->asic_type == CHIP_KAVERI) {
2703                 const struct gfx_firmware_header_v1_0 *mec2_hdr;
2704
2705                 if (!adev->gfx.mec2_fw)
2706                         return -EINVAL;
2707
2708                 mec2_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec2_fw->data;
2709                 amdgpu_ucode_print_gfx_hdr(&mec2_hdr->header);
2710                 adev->gfx.mec2_fw_version = le32_to_cpu(mec2_hdr->header.ucode_version);
2711                 adev->gfx.mec2_feature_version = le32_to_cpu(
2712                                 mec2_hdr->ucode_feature_version);
2713
2714                 /* MEC2 */
2715                 fw_data = (const __le32 *)
2716                         (adev->gfx.mec2_fw->data +
2717                          le32_to_cpu(mec2_hdr->header.ucode_array_offset_bytes));
2718                 fw_size = le32_to_cpu(mec2_hdr->header.ucode_size_bytes) / 4;
2719                 WREG32(mmCP_MEC_ME2_UCODE_ADDR, 0);
2720                 for (i = 0; i < fw_size; i++)
2721                         WREG32(mmCP_MEC_ME2_UCODE_DATA, le32_to_cpup(fw_data++));
2722                 WREG32(mmCP_MEC_ME2_UCODE_ADDR, 0);
2723         }
2724
2725         return 0;
2726 }
2727
2728 /**
2729  * gfx_v7_0_cp_compute_fini - stop the compute queues
2730  *
2731  * @adev: amdgpu_device pointer
2732  *
2733  * Stop the compute queues and tear down the driver queue
2734  * info.
2735  */
2736 static void gfx_v7_0_cp_compute_fini(struct amdgpu_device *adev)
2737 {
2738         int i;
2739
2740         for (i = 0; i < adev->gfx.num_compute_rings; i++) {
2741                 struct amdgpu_ring *ring = &adev->gfx.compute_ring[i];
2742
2743                 amdgpu_bo_free_kernel(&ring->mqd_obj, NULL, NULL);
2744         }
2745 }
2746
2747 static void gfx_v7_0_mec_fini(struct amdgpu_device *adev)
2748 {
2749         amdgpu_bo_free_kernel(&adev->gfx.mec.hpd_eop_obj, NULL, NULL);
2750 }
2751
2752 static int gfx_v7_0_mec_init(struct amdgpu_device *adev)
2753 {
2754         int r;
2755         u32 *hpd;
2756         size_t mec_hpd_size;
2757
2758         bitmap_zero(adev->gfx.mec.queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);
2759
2760         /* take ownership of the relevant compute queues */
2761         amdgpu_gfx_compute_queue_acquire(adev);
2762
2763         /* allocate space for ALL pipes (even the ones we don't own) */
2764         mec_hpd_size = adev->gfx.mec.num_mec * adev->gfx.mec.num_pipe_per_mec
2765                 * GFX7_MEC_HPD_SIZE * 2;
2766
2767         r = amdgpu_bo_create_reserved(adev, mec_hpd_size, PAGE_SIZE,
2768                                       AMDGPU_GEM_DOMAIN_VRAM,
2769                                       &adev->gfx.mec.hpd_eop_obj,
2770                                       &adev->gfx.mec.hpd_eop_gpu_addr,
2771                                       (void **)&hpd);
2772         if (r) {
2773                 dev_warn(adev->dev, "(%d) create, pin or map of HDP EOP bo failed\n", r);
2774                 gfx_v7_0_mec_fini(adev);
2775                 return r;
2776         }
2777
2778         /* clear memory.  Not sure if this is required or not */
2779         memset(hpd, 0, mec_hpd_size);
2780
2781         amdgpu_bo_kunmap(adev->gfx.mec.hpd_eop_obj);
2782         amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj);
2783
2784         return 0;
2785 }
2786
2787 struct hqd_registers
2788 {
2789         u32 cp_mqd_base_addr;
2790         u32 cp_mqd_base_addr_hi;
2791         u32 cp_hqd_active;
2792         u32 cp_hqd_vmid;
2793         u32 cp_hqd_persistent_state;
2794         u32 cp_hqd_pipe_priority;
2795         u32 cp_hqd_queue_priority;
2796         u32 cp_hqd_quantum;
2797         u32 cp_hqd_pq_base;
2798         u32 cp_hqd_pq_base_hi;
2799         u32 cp_hqd_pq_rptr;
2800         u32 cp_hqd_pq_rptr_report_addr;
2801         u32 cp_hqd_pq_rptr_report_addr_hi;
2802         u32 cp_hqd_pq_wptr_poll_addr;
2803         u32 cp_hqd_pq_wptr_poll_addr_hi;
2804         u32 cp_hqd_pq_doorbell_control;
2805         u32 cp_hqd_pq_wptr;
2806         u32 cp_hqd_pq_control;
2807         u32 cp_hqd_ib_base_addr;
2808         u32 cp_hqd_ib_base_addr_hi;
2809         u32 cp_hqd_ib_rptr;
2810         u32 cp_hqd_ib_control;
2811         u32 cp_hqd_iq_timer;
2812         u32 cp_hqd_iq_rptr;
2813         u32 cp_hqd_dequeue_request;
2814         u32 cp_hqd_dma_offload;
2815         u32 cp_hqd_sema_cmd;
2816         u32 cp_hqd_msg_type;
2817         u32 cp_hqd_atomic0_preop_lo;
2818         u32 cp_hqd_atomic0_preop_hi;
2819         u32 cp_hqd_atomic1_preop_lo;
2820         u32 cp_hqd_atomic1_preop_hi;
2821         u32 cp_hqd_hq_scheduler0;
2822         u32 cp_hqd_hq_scheduler1;
2823         u32 cp_mqd_control;
2824 };
2825
2826 static void gfx_v7_0_compute_pipe_init(struct amdgpu_device *adev,
2827                                        int mec, int pipe)
2828 {
2829         u64 eop_gpu_addr;
2830         u32 tmp;
2831         size_t eop_offset = (mec * adev->gfx.mec.num_pipe_per_mec + pipe)
2832                             * GFX7_MEC_HPD_SIZE * 2;
2833
2834         mutex_lock(&adev->srbm_mutex);
2835         eop_gpu_addr = adev->gfx.mec.hpd_eop_gpu_addr + eop_offset;
2836
2837         cik_srbm_select(adev, mec + 1, pipe, 0, 0);
2838
2839         /* write the EOP addr */
2840         WREG32(mmCP_HPD_EOP_BASE_ADDR, eop_gpu_addr >> 8);
2841         WREG32(mmCP_HPD_EOP_BASE_ADDR_HI, upper_32_bits(eop_gpu_addr) >> 8);
2842
2843         /* set the VMID assigned */
2844         WREG32(mmCP_HPD_EOP_VMID, 0);
2845
2846         /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
2847         tmp = RREG32(mmCP_HPD_EOP_CONTROL);
2848         tmp &= ~CP_HPD_EOP_CONTROL__EOP_SIZE_MASK;
2849         tmp |= order_base_2(GFX7_MEC_HPD_SIZE / 8);
2850         WREG32(mmCP_HPD_EOP_CONTROL, tmp);
2851
2852         cik_srbm_select(adev, 0, 0, 0, 0);
2853         mutex_unlock(&adev->srbm_mutex);
2854 }
2855
2856 static int gfx_v7_0_mqd_deactivate(struct amdgpu_device *adev)
2857 {
2858         int i;
2859
2860         /* disable the queue if it's active */
2861         if (RREG32(mmCP_HQD_ACTIVE) & 1) {
2862                 WREG32(mmCP_HQD_DEQUEUE_REQUEST, 1);
2863                 for (i = 0; i < adev->usec_timeout; i++) {
2864                         if (!(RREG32(mmCP_HQD_ACTIVE) & 1))
2865                                 break;
2866                         udelay(1);
2867                 }
2868
2869                 if (i == adev->usec_timeout)
2870                         return -ETIMEDOUT;
2871
2872                 WREG32(mmCP_HQD_DEQUEUE_REQUEST, 0);
2873                 WREG32(mmCP_HQD_PQ_RPTR, 0);
2874                 WREG32(mmCP_HQD_PQ_WPTR, 0);
2875         }
2876
2877         return 0;
2878 }
2879
2880 static void gfx_v7_0_mqd_init(struct amdgpu_device *adev,
2881                              struct cik_mqd *mqd,
2882                              uint64_t mqd_gpu_addr,
2883                              struct amdgpu_ring *ring)
2884 {
2885         u64 hqd_gpu_addr;
2886         u64 wb_gpu_addr;
2887
2888         /* init the mqd struct */
2889         memset(mqd, 0, sizeof(struct cik_mqd));
2890
2891         mqd->header = 0xC0310800;
2892         mqd->compute_static_thread_mgmt_se0 = 0xffffffff;
2893         mqd->compute_static_thread_mgmt_se1 = 0xffffffff;
2894         mqd->compute_static_thread_mgmt_se2 = 0xffffffff;
2895         mqd->compute_static_thread_mgmt_se3 = 0xffffffff;
2896
2897         /* enable doorbell? */
2898         mqd->cp_hqd_pq_doorbell_control =
2899                 RREG32(mmCP_HQD_PQ_DOORBELL_CONTROL);
2900         if (ring->use_doorbell)
2901                 mqd->cp_hqd_pq_doorbell_control |= CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_EN_MASK;
2902         else
2903                 mqd->cp_hqd_pq_doorbell_control &= ~CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_EN_MASK;
2904
2905         /* set the pointer to the MQD */
2906         mqd->cp_mqd_base_addr_lo = mqd_gpu_addr & 0xfffffffc;
2907         mqd->cp_mqd_base_addr_hi = upper_32_bits(mqd_gpu_addr);
2908
2909         /* set MQD vmid to 0 */
2910         mqd->cp_mqd_control = RREG32(mmCP_MQD_CONTROL);
2911         mqd->cp_mqd_control &= ~CP_MQD_CONTROL__VMID_MASK;
2912
2913         /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
2914         hqd_gpu_addr = ring->gpu_addr >> 8;
2915         mqd->cp_hqd_pq_base_lo = hqd_gpu_addr;
2916         mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
2917
2918         /* set up the HQD, this is similar to CP_RB0_CNTL */
2919         mqd->cp_hqd_pq_control = RREG32(mmCP_HQD_PQ_CONTROL);
2920         mqd->cp_hqd_pq_control &=
2921                 ~(CP_HQD_PQ_CONTROL__QUEUE_SIZE_MASK |
2922                                 CP_HQD_PQ_CONTROL__RPTR_BLOCK_SIZE_MASK);
2923
2924         mqd->cp_hqd_pq_control |=
2925                 order_base_2(ring->ring_size / 8);
2926         mqd->cp_hqd_pq_control |=
2927                 (order_base_2(AMDGPU_GPU_PAGE_SIZE/8) << 8);
2928 #ifdef __BIG_ENDIAN
2929         mqd->cp_hqd_pq_control |=
2930                 2 << CP_HQD_PQ_CONTROL__ENDIAN_SWAP__SHIFT;
2931 #endif
2932         mqd->cp_hqd_pq_control &=
2933                 ~(CP_HQD_PQ_CONTROL__UNORD_DISPATCH_MASK |
2934                                 CP_HQD_PQ_CONTROL__ROQ_PQ_IB_FLIP_MASK |
2935                                 CP_HQD_PQ_CONTROL__PQ_VOLATILE_MASK);
2936         mqd->cp_hqd_pq_control |=
2937                 CP_HQD_PQ_CONTROL__PRIV_STATE_MASK |
2938                 CP_HQD_PQ_CONTROL__KMD_QUEUE_MASK; /* assuming kernel queue control */
2939
2940         /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
2941         wb_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4);
2942         mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc;
2943         mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
2944
2945         /* set the wb address wether it's enabled or not */
2946         wb_gpu_addr = adev->wb.gpu_addr + (ring->rptr_offs * 4);
2947         mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc;
2948         mqd->cp_hqd_pq_rptr_report_addr_hi =
2949                 upper_32_bits(wb_gpu_addr) & 0xffff;
2950
2951         /* enable the doorbell if requested */
2952         if (ring->use_doorbell) {
2953                 mqd->cp_hqd_pq_doorbell_control =
2954                         RREG32(mmCP_HQD_PQ_DOORBELL_CONTROL);
2955                 mqd->cp_hqd_pq_doorbell_control &=
2956                         ~CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET_MASK;
2957                 mqd->cp_hqd_pq_doorbell_control |=
2958                         (ring->doorbell_index <<
2959                          CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET__SHIFT);
2960                 mqd->cp_hqd_pq_doorbell_control |=
2961                         CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_EN_MASK;
2962                 mqd->cp_hqd_pq_doorbell_control &=
2963                         ~(CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_SOURCE_MASK |
2964                                         CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_HIT_MASK);
2965
2966         } else {
2967                 mqd->cp_hqd_pq_doorbell_control = 0;
2968         }
2969
2970         /* read and write pointers, similar to CP_RB0_WPTR/_RPTR */
2971         ring->wptr = 0;
2972         mqd->cp_hqd_pq_wptr = lower_32_bits(ring->wptr);
2973         mqd->cp_hqd_pq_rptr = RREG32(mmCP_HQD_PQ_RPTR);
2974
2975         /* set the vmid for the queue */
2976         mqd->cp_hqd_vmid = 0;
2977
2978         /* defaults */
2979         mqd->cp_hqd_ib_control = RREG32(mmCP_HQD_IB_CONTROL);
2980         mqd->cp_hqd_ib_base_addr_lo = RREG32(mmCP_HQD_IB_BASE_ADDR);
2981         mqd->cp_hqd_ib_base_addr_hi = RREG32(mmCP_HQD_IB_BASE_ADDR_HI);
2982         mqd->cp_hqd_ib_rptr = RREG32(mmCP_HQD_IB_RPTR);
2983         mqd->cp_hqd_persistent_state = RREG32(mmCP_HQD_PERSISTENT_STATE);
2984         mqd->cp_hqd_sema_cmd = RREG32(mmCP_HQD_SEMA_CMD);
2985         mqd->cp_hqd_msg_type = RREG32(mmCP_HQD_MSG_TYPE);
2986         mqd->cp_hqd_atomic0_preop_lo = RREG32(mmCP_HQD_ATOMIC0_PREOP_LO);
2987         mqd->cp_hqd_atomic0_preop_hi = RREG32(mmCP_HQD_ATOMIC0_PREOP_HI);
2988         mqd->cp_hqd_atomic1_preop_lo = RREG32(mmCP_HQD_ATOMIC1_PREOP_LO);
2989         mqd->cp_hqd_atomic1_preop_hi = RREG32(mmCP_HQD_ATOMIC1_PREOP_HI);
2990         mqd->cp_hqd_pq_rptr = RREG32(mmCP_HQD_PQ_RPTR);
2991         mqd->cp_hqd_quantum = RREG32(mmCP_HQD_QUANTUM);
2992         mqd->cp_hqd_pipe_priority = RREG32(mmCP_HQD_PIPE_PRIORITY);
2993         mqd->cp_hqd_queue_priority = RREG32(mmCP_HQD_QUEUE_PRIORITY);
2994         mqd->cp_hqd_iq_rptr = RREG32(mmCP_HQD_IQ_RPTR);
2995
2996         /* activate the queue */
2997         mqd->cp_hqd_active = 1;
2998 }
2999
3000 int gfx_v7_0_mqd_commit(struct amdgpu_device *adev, struct cik_mqd *mqd)
3001 {
3002         uint32_t tmp;
3003         uint32_t mqd_reg;
3004         uint32_t *mqd_data;
3005
3006         /* HQD registers extend from mmCP_MQD_BASE_ADDR to mmCP_MQD_CONTROL */
3007         mqd_data = &mqd->cp_mqd_base_addr_lo;
3008
3009         /* disable wptr polling */
3010         tmp = RREG32(mmCP_PQ_WPTR_POLL_CNTL);
3011         tmp = REG_SET_FIELD(tmp, CP_PQ_WPTR_POLL_CNTL, EN, 0);
3012         WREG32(mmCP_PQ_WPTR_POLL_CNTL, tmp);
3013
3014         /* program all HQD registers */
3015         for (mqd_reg = mmCP_HQD_VMID; mqd_reg <= mmCP_MQD_CONTROL; mqd_reg++)
3016                 WREG32(mqd_reg, mqd_data[mqd_reg - mmCP_MQD_BASE_ADDR]);
3017
3018         /* activate the HQD */
3019         for (mqd_reg = mmCP_MQD_BASE_ADDR; mqd_reg <= mmCP_HQD_ACTIVE; mqd_reg++)
3020                 WREG32(mqd_reg, mqd_data[mqd_reg - mmCP_MQD_BASE_ADDR]);
3021
3022         return 0;
3023 }
3024
3025 static int gfx_v7_0_compute_queue_init(struct amdgpu_device *adev, int ring_id)
3026 {
3027         int r;
3028         u64 mqd_gpu_addr;
3029         struct cik_mqd *mqd;
3030         struct amdgpu_ring *ring = &adev->gfx.compute_ring[ring_id];
3031
3032         r = amdgpu_bo_create_reserved(adev, sizeof(struct cik_mqd), PAGE_SIZE,
3033                                       AMDGPU_GEM_DOMAIN_GTT, &ring->mqd_obj,
3034                                       &mqd_gpu_addr, (void **)&mqd);
3035         if (r) {
3036                 dev_warn(adev->dev, "(%d) create MQD bo failed\n", r);
3037                 return r;
3038         }
3039
3040         mutex_lock(&adev->srbm_mutex);
3041         cik_srbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
3042
3043         gfx_v7_0_mqd_init(adev, mqd, mqd_gpu_addr, ring);
3044         gfx_v7_0_mqd_deactivate(adev);
3045         gfx_v7_0_mqd_commit(adev, mqd);
3046
3047         cik_srbm_select(adev, 0, 0, 0, 0);
3048         mutex_unlock(&adev->srbm_mutex);
3049
3050         amdgpu_bo_kunmap(ring->mqd_obj);
3051         amdgpu_bo_unreserve(ring->mqd_obj);
3052         return 0;
3053 }
3054
3055 /**
3056  * gfx_v7_0_cp_compute_resume - setup the compute queue registers
3057  *
3058  * @adev: amdgpu_device pointer
3059  *
3060  * Program the compute queues and test them to make sure they
3061  * are working.
3062  * Returns 0 for success, error for failure.
3063  */
3064 static int gfx_v7_0_cp_compute_resume(struct amdgpu_device *adev)
3065 {
3066         int r, i, j;
3067         u32 tmp;
3068         struct amdgpu_ring *ring;
3069
3070         /* fix up chicken bits */
3071         tmp = RREG32(mmCP_CPF_DEBUG);
3072         tmp |= (1 << 23);
3073         WREG32(mmCP_CPF_DEBUG, tmp);
3074
3075         /* init all pipes (even the ones we don't own) */
3076         for (i = 0; i < adev->gfx.mec.num_mec; i++)
3077                 for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++)
3078                         gfx_v7_0_compute_pipe_init(adev, i, j);
3079
3080         /* init the queues */
3081         for (i = 0; i < adev->gfx.num_compute_rings; i++) {
3082                 r = gfx_v7_0_compute_queue_init(adev, i);
3083                 if (r) {
3084                         gfx_v7_0_cp_compute_fini(adev);
3085                         return r;
3086                 }
3087         }
3088
3089         gfx_v7_0_cp_compute_enable(adev, true);
3090
3091         for (i = 0; i < adev->gfx.num_compute_rings; i++) {
3092                 ring = &adev->gfx.compute_ring[i];
3093                 amdgpu_ring_test_helper(ring);
3094         }
3095
3096         return 0;
3097 }
3098
3099 static void gfx_v7_0_cp_enable(struct amdgpu_device *adev, bool enable)
3100 {
3101         gfx_v7_0_cp_gfx_enable(adev, enable);
3102         gfx_v7_0_cp_compute_enable(adev, enable);
3103 }
3104
3105 static int gfx_v7_0_cp_load_microcode(struct amdgpu_device *adev)
3106 {
3107         int r;
3108
3109         r = gfx_v7_0_cp_gfx_load_microcode(adev);
3110         if (r)
3111                 return r;
3112         r = gfx_v7_0_cp_compute_load_microcode(adev);
3113         if (r)
3114                 return r;
3115
3116         return 0;
3117 }
3118
3119 static void gfx_v7_0_enable_gui_idle_interrupt(struct amdgpu_device *adev,
3120                                                bool enable)
3121 {
3122         u32 tmp = RREG32(mmCP_INT_CNTL_RING0);
3123
3124         if (enable)
3125                 tmp |= (CP_INT_CNTL_RING0__CNTX_BUSY_INT_ENABLE_MASK |
3126                                 CP_INT_CNTL_RING0__CNTX_EMPTY_INT_ENABLE_MASK);
3127         else
3128                 tmp &= ~(CP_INT_CNTL_RING0__CNTX_BUSY_INT_ENABLE_MASK |
3129                                 CP_INT_CNTL_RING0__CNTX_EMPTY_INT_ENABLE_MASK);
3130         WREG32(mmCP_INT_CNTL_RING0, tmp);
3131 }
3132
3133 static int gfx_v7_0_cp_resume(struct amdgpu_device *adev)
3134 {
3135         int r;
3136
3137         gfx_v7_0_enable_gui_idle_interrupt(adev, false);
3138
3139         r = gfx_v7_0_cp_load_microcode(adev);
3140         if (r)
3141                 return r;
3142
3143         r = gfx_v7_0_cp_gfx_resume(adev);
3144         if (r)
3145                 return r;
3146         r = gfx_v7_0_cp_compute_resume(adev);
3147         if (r)
3148                 return r;
3149
3150         gfx_v7_0_enable_gui_idle_interrupt(adev, true);
3151
3152         return 0;
3153 }
3154
3155 /**
3156  * gfx_v7_0_ring_emit_vm_flush - cik vm flush using the CP
3157  *
3158  * @ring: the ring to emmit the commands to
3159  *
3160  * Sync the command pipeline with the PFP. E.g. wait for everything
3161  * to be completed.
3162  */
3163 static void gfx_v7_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
3164 {
3165         int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
3166         uint32_t seq = ring->fence_drv.sync_seq;
3167         uint64_t addr = ring->fence_drv.gpu_addr;
3168
3169         amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
3170         amdgpu_ring_write(ring, (WAIT_REG_MEM_MEM_SPACE(1) | /* memory */
3171                                  WAIT_REG_MEM_FUNCTION(3) | /* equal */
3172                                  WAIT_REG_MEM_ENGINE(usepfp)));   /* pfp or me */
3173         amdgpu_ring_write(ring, addr & 0xfffffffc);
3174         amdgpu_ring_write(ring, upper_32_bits(addr) & 0xffffffff);
3175         amdgpu_ring_write(ring, seq);
3176         amdgpu_ring_write(ring, 0xffffffff);
3177         amdgpu_ring_write(ring, 4); /* poll interval */
3178
3179         if (usepfp) {
3180                 /* synce CE with ME to prevent CE fetch CEIB before context switch done */
3181                 amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
3182                 amdgpu_ring_write(ring, 0);
3183                 amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
3184                 amdgpu_ring_write(ring, 0);
3185         }
3186 }
3187
3188 /*
3189  * vm
3190  * VMID 0 is the physical GPU addresses as used by the kernel.
3191  * VMIDs 1-15 are used for userspace clients and are handled
3192  * by the amdgpu vm/hsa code.
3193  */
3194 /**
3195  * gfx_v7_0_ring_emit_vm_flush - cik vm flush using the CP
3196  *
3197  * @adev: amdgpu_device pointer
3198  *
3199  * Update the page table base and flush the VM TLB
3200  * using the CP (CIK).
3201  */
3202 static void gfx_v7_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
3203                                         unsigned vmid, uint64_t pd_addr)
3204 {
3205         int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
3206
3207         amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
3208
3209         /* wait for the invalidate to complete */
3210         amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
3211         amdgpu_ring_write(ring, (WAIT_REG_MEM_OPERATION(0) | /* wait */
3212                                  WAIT_REG_MEM_FUNCTION(0) |  /* always */
3213                                  WAIT_REG_MEM_ENGINE(0))); /* me */
3214         amdgpu_ring_write(ring, mmVM_INVALIDATE_REQUEST);
3215         amdgpu_ring_write(ring, 0);
3216         amdgpu_ring_write(ring, 0); /* ref */
3217         amdgpu_ring_write(ring, 0); /* mask */
3218         amdgpu_ring_write(ring, 0x20); /* poll interval */
3219
3220         /* compute doesn't have PFP */
3221         if (usepfp) {
3222                 /* sync PFP to ME, otherwise we might get invalid PFP reads */
3223                 amdgpu_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
3224                 amdgpu_ring_write(ring, 0x0);
3225
3226                 /* synce CE with ME to prevent CE fetch CEIB before context switch done */
3227                 amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
3228                 amdgpu_ring_write(ring, 0);
3229                 amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
3230                 amdgpu_ring_write(ring, 0);
3231         }
3232 }
3233
3234 static void gfx_v7_0_ring_emit_wreg(struct amdgpu_ring *ring,
3235                                     uint32_t reg, uint32_t val)
3236 {
3237         int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
3238
3239         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3240         amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
3241                                  WRITE_DATA_DST_SEL(0)));
3242         amdgpu_ring_write(ring, reg);
3243         amdgpu_ring_write(ring, 0);
3244         amdgpu_ring_write(ring, val);
3245 }
3246
3247 /*
3248  * RLC
3249  * The RLC is a multi-purpose microengine that handles a
3250  * variety of functions.
3251  */
3252 static int gfx_v7_0_rlc_init(struct amdgpu_device *adev)
3253 {
3254         const u32 *src_ptr;
3255         u32 dws;
3256         const struct cs_section_def *cs_data;
3257         int r;
3258
3259         /* allocate rlc buffers */
3260         if (adev->flags & AMD_IS_APU) {
3261                 if (adev->asic_type == CHIP_KAVERI) {
3262                         adev->gfx.rlc.reg_list = spectre_rlc_save_restore_register_list;
3263                         adev->gfx.rlc.reg_list_size =
3264                                 (u32)ARRAY_SIZE(spectre_rlc_save_restore_register_list);
3265                 } else {
3266                         adev->gfx.rlc.reg_list = kalindi_rlc_save_restore_register_list;
3267                         adev->gfx.rlc.reg_list_size =
3268                                 (u32)ARRAY_SIZE(kalindi_rlc_save_restore_register_list);
3269                 }
3270         }
3271         adev->gfx.rlc.cs_data = ci_cs_data;
3272         adev->gfx.rlc.cp_table_size = ALIGN(CP_ME_TABLE_SIZE * 5 * 4, 2048); /* CP JT */
3273         adev->gfx.rlc.cp_table_size += 64 * 1024; /* GDS */
3274
3275         src_ptr = adev->gfx.rlc.reg_list;
3276         dws = adev->gfx.rlc.reg_list_size;
3277         dws += (5 * 16) + 48 + 48 + 64;
3278
3279         cs_data = adev->gfx.rlc.cs_data;
3280
3281         if (src_ptr) {
3282                 /* init save restore block */
3283                 r = amdgpu_gfx_rlc_init_sr(adev, dws);
3284                 if (r)
3285                         return r;
3286         }
3287
3288         if (cs_data) {
3289                 /* init clear state block */
3290                 r = amdgpu_gfx_rlc_init_csb(adev);
3291                 if (r)
3292                         return r;
3293         }
3294
3295         if (adev->gfx.rlc.cp_table_size) {
3296                 r = amdgpu_gfx_rlc_init_cpt(adev);
3297                 if (r)
3298                         return r;
3299         }
3300
3301         return 0;
3302 }
3303
3304 static void gfx_v7_0_enable_lbpw(struct amdgpu_device *adev, bool enable)
3305 {
3306         u32 tmp;
3307
3308         tmp = RREG32(mmRLC_LB_CNTL);
3309         if (enable)
3310                 tmp |= RLC_LB_CNTL__LOAD_BALANCE_ENABLE_MASK;
3311         else
3312                 tmp &= ~RLC_LB_CNTL__LOAD_BALANCE_ENABLE_MASK;
3313         WREG32(mmRLC_LB_CNTL, tmp);
3314 }
3315
3316 static void gfx_v7_0_wait_for_rlc_serdes(struct amdgpu_device *adev)
3317 {
3318         u32 i, j, k;
3319         u32 mask;
3320
3321         mutex_lock(&adev->grbm_idx_mutex);
3322         for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
3323                 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
3324                         gfx_v7_0_select_se_sh(adev, i, j, 0xffffffff);
3325                         for (k = 0; k < adev->usec_timeout; k++) {
3326                                 if (RREG32(mmRLC_SERDES_CU_MASTER_BUSY) == 0)
3327                                         break;
3328                                 udelay(1);
3329                         }
3330                 }
3331         }
3332         gfx_v7_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
3333         mutex_unlock(&adev->grbm_idx_mutex);
3334
3335         mask = RLC_SERDES_NONCU_MASTER_BUSY__SE_MASTER_BUSY_MASK |
3336                 RLC_SERDES_NONCU_MASTER_BUSY__GC_MASTER_BUSY_MASK |
3337                 RLC_SERDES_NONCU_MASTER_BUSY__TC0_MASTER_BUSY_MASK |
3338                 RLC_SERDES_NONCU_MASTER_BUSY__TC1_MASTER_BUSY_MASK;
3339         for (k = 0; k < adev->usec_timeout; k++) {
3340                 if ((RREG32(mmRLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)
3341                         break;
3342                 udelay(1);
3343         }
3344 }
3345
3346 static void gfx_v7_0_update_rlc(struct amdgpu_device *adev, u32 rlc)
3347 {
3348         u32 tmp;
3349
3350         tmp = RREG32(mmRLC_CNTL);
3351         if (tmp != rlc)
3352                 WREG32(mmRLC_CNTL, rlc);
3353 }
3354
3355 static u32 gfx_v7_0_halt_rlc(struct amdgpu_device *adev)
3356 {
3357         u32 data, orig;
3358
3359         orig = data = RREG32(mmRLC_CNTL);
3360
3361         if (data & RLC_CNTL__RLC_ENABLE_F32_MASK) {
3362                 u32 i;
3363
3364                 data &= ~RLC_CNTL__RLC_ENABLE_F32_MASK;
3365                 WREG32(mmRLC_CNTL, data);
3366
3367                 for (i = 0; i < adev->usec_timeout; i++) {
3368                         if ((RREG32(mmRLC_GPM_STAT) & RLC_GPM_STAT__RLC_BUSY_MASK) == 0)
3369                                 break;
3370                         udelay(1);
3371                 }
3372
3373                 gfx_v7_0_wait_for_rlc_serdes(adev);
3374         }
3375
3376         return orig;
3377 }
3378
3379 static bool gfx_v7_0_is_rlc_enabled(struct amdgpu_device *adev)
3380 {
3381         return true;
3382 }
3383
3384 static void gfx_v7_0_set_safe_mode(struct amdgpu_device *adev)
3385 {
3386         u32 tmp, i, mask;
3387
3388         tmp = 0x1 | (1 << 1);
3389         WREG32(mmRLC_GPR_REG2, tmp);
3390
3391         mask = RLC_GPM_STAT__GFX_POWER_STATUS_MASK |
3392                 RLC_GPM_STAT__GFX_CLOCK_STATUS_MASK;
3393         for (i = 0; i < adev->usec_timeout; i++) {
3394                 if ((RREG32(mmRLC_GPM_STAT) & mask) == mask)
3395                         break;
3396                 udelay(1);
3397         }
3398
3399         for (i = 0; i < adev->usec_timeout; i++) {
3400                 if ((RREG32(mmRLC_GPR_REG2) & 0x1) == 0)
3401                         break;
3402                 udelay(1);
3403         }
3404 }
3405
3406 static void gfx_v7_0_unset_safe_mode(struct amdgpu_device *adev)
3407 {
3408         u32 tmp;
3409
3410         tmp = 0x1 | (0 << 1);
3411         WREG32(mmRLC_GPR_REG2, tmp);
3412 }
3413
3414 /**
3415  * gfx_v7_0_rlc_stop - stop the RLC ME
3416  *
3417  * @adev: amdgpu_device pointer
3418  *
3419  * Halt the RLC ME (MicroEngine) (CIK).
3420  */
3421 static void gfx_v7_0_rlc_stop(struct amdgpu_device *adev)
3422 {
3423         WREG32(mmRLC_CNTL, 0);
3424
3425         gfx_v7_0_enable_gui_idle_interrupt(adev, false);
3426
3427         gfx_v7_0_wait_for_rlc_serdes(adev);
3428 }
3429
3430 /**
3431  * gfx_v7_0_rlc_start - start the RLC ME
3432  *
3433  * @adev: amdgpu_device pointer
3434  *
3435  * Unhalt the RLC ME (MicroEngine) (CIK).
3436  */
3437 static void gfx_v7_0_rlc_start(struct amdgpu_device *adev)
3438 {
3439         WREG32(mmRLC_CNTL, RLC_CNTL__RLC_ENABLE_F32_MASK);
3440
3441         gfx_v7_0_enable_gui_idle_interrupt(adev, true);
3442
3443         udelay(50);
3444 }
3445
3446 static void gfx_v7_0_rlc_reset(struct amdgpu_device *adev)
3447 {
3448         u32 tmp = RREG32(mmGRBM_SOFT_RESET);
3449
3450         tmp |= GRBM_SOFT_RESET__SOFT_RESET_RLC_MASK;
3451         WREG32(mmGRBM_SOFT_RESET, tmp);
3452         udelay(50);
3453         tmp &= ~GRBM_SOFT_RESET__SOFT_RESET_RLC_MASK;
3454         WREG32(mmGRBM_SOFT_RESET, tmp);
3455         udelay(50);
3456 }
3457
3458 /**
3459  * gfx_v7_0_rlc_resume - setup the RLC hw
3460  *
3461  * @adev: amdgpu_device pointer
3462  *
3463  * Initialize the RLC registers, load the ucode,
3464  * and start the RLC (CIK).
3465  * Returns 0 for success, -EINVAL if the ucode is not available.
3466  */
3467 static int gfx_v7_0_rlc_resume(struct amdgpu_device *adev)
3468 {
3469         const struct rlc_firmware_header_v1_0 *hdr;
3470         const __le32 *fw_data;
3471         unsigned i, fw_size;
3472         u32 tmp;
3473
3474         if (!adev->gfx.rlc_fw)
3475                 return -EINVAL;
3476
3477         hdr = (const struct rlc_firmware_header_v1_0 *)adev->gfx.rlc_fw->data;
3478         amdgpu_ucode_print_rlc_hdr(&hdr->header);
3479         adev->gfx.rlc_fw_version = le32_to_cpu(hdr->header.ucode_version);
3480         adev->gfx.rlc_feature_version = le32_to_cpu(
3481                                         hdr->ucode_feature_version);
3482
3483         adev->gfx.rlc.funcs->stop(adev);
3484
3485         /* disable CG */
3486         tmp = RREG32(mmRLC_CGCG_CGLS_CTRL) & 0xfffffffc;
3487         WREG32(mmRLC_CGCG_CGLS_CTRL, tmp);
3488
3489         adev->gfx.rlc.funcs->reset(adev);
3490
3491         gfx_v7_0_init_pg(adev);
3492
3493         WREG32(mmRLC_LB_CNTR_INIT, 0);
3494         WREG32(mmRLC_LB_CNTR_MAX, 0x00008000);
3495
3496         mutex_lock(&adev->grbm_idx_mutex);
3497         gfx_v7_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
3498         WREG32(mmRLC_LB_INIT_CU_MASK, 0xffffffff);
3499         WREG32(mmRLC_LB_PARAMS, 0x00600408);
3500         WREG32(mmRLC_LB_CNTL, 0x80000004);
3501         mutex_unlock(&adev->grbm_idx_mutex);
3502
3503         WREG32(mmRLC_MC_CNTL, 0);
3504         WREG32(mmRLC_UCODE_CNTL, 0);
3505
3506         fw_data = (const __le32 *)
3507                 (adev->gfx.rlc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
3508         fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
3509         WREG32(mmRLC_GPM_UCODE_ADDR, 0);
3510         for (i = 0; i < fw_size; i++)
3511                 WREG32(mmRLC_GPM_UCODE_DATA, le32_to_cpup(fw_data++));
3512         WREG32(mmRLC_GPM_UCODE_ADDR, adev->gfx.rlc_fw_version);
3513
3514         /* XXX - find out what chips support lbpw */
3515         gfx_v7_0_enable_lbpw(adev, false);
3516
3517         if (adev->asic_type == CHIP_BONAIRE)
3518                 WREG32(mmRLC_DRIVER_CPDMA_STATUS, 0);
3519
3520         adev->gfx.rlc.funcs->start(adev);
3521
3522         return 0;
3523 }
3524
3525 static void gfx_v7_0_enable_cgcg(struct amdgpu_device *adev, bool enable)
3526 {
3527         u32 data, orig, tmp, tmp2;
3528
3529         orig = data = RREG32(mmRLC_CGCG_CGLS_CTRL);
3530
3531         if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)) {
3532                 gfx_v7_0_enable_gui_idle_interrupt(adev, true);
3533
3534                 tmp = gfx_v7_0_halt_rlc(adev);
3535
3536                 mutex_lock(&adev->grbm_idx_mutex);
3537                 gfx_v7_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
3538                 WREG32(mmRLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
3539                 WREG32(mmRLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
3540                 tmp2 = RLC_SERDES_WR_CTRL__BPM_ADDR_MASK |
3541                         RLC_SERDES_WR_CTRL__CGCG_OVERRIDE_0_MASK |
3542                         RLC_SERDES_WR_CTRL__CGLS_ENABLE_MASK;
3543                 WREG32(mmRLC_SERDES_WR_CTRL, tmp2);
3544                 mutex_unlock(&adev->grbm_idx_mutex);
3545
3546                 gfx_v7_0_update_rlc(adev, tmp);
3547
3548                 data |= RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK | RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK;
3549                 if (orig != data)
3550                         WREG32(mmRLC_CGCG_CGLS_CTRL, data);
3551
3552         } else {
3553                 gfx_v7_0_enable_gui_idle_interrupt(adev, false);
3554
3555                 RREG32(mmCB_CGTT_SCLK_CTRL);
3556                 RREG32(mmCB_CGTT_SCLK_CTRL);
3557                 RREG32(mmCB_CGTT_SCLK_CTRL);
3558                 RREG32(mmCB_CGTT_SCLK_CTRL);
3559
3560                 data &= ~(RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK | RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK);
3561                 if (orig != data)
3562                         WREG32(mmRLC_CGCG_CGLS_CTRL, data);
3563
3564                 gfx_v7_0_enable_gui_idle_interrupt(adev, true);
3565         }
3566 }
3567
3568 static void gfx_v7_0_enable_mgcg(struct amdgpu_device *adev, bool enable)
3569 {
3570         u32 data, orig, tmp = 0;
3571
3572         if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG)) {
3573                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) {
3574                         if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CP_LS) {
3575                                 orig = data = RREG32(mmCP_MEM_SLP_CNTL);
3576                                 data |= CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK;
3577                                 if (orig != data)
3578                                         WREG32(mmCP_MEM_SLP_CNTL, data);
3579                         }
3580                 }
3581
3582                 orig = data = RREG32(mmRLC_CGTT_MGCG_OVERRIDE);
3583                 data |= 0x00000001;
3584                 data &= 0xfffffffd;
3585                 if (orig != data)
3586                         WREG32(mmRLC_CGTT_MGCG_OVERRIDE, data);
3587
3588                 tmp = gfx_v7_0_halt_rlc(adev);
3589
3590                 mutex_lock(&adev->grbm_idx_mutex);
3591                 gfx_v7_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
3592                 WREG32(mmRLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
3593                 WREG32(mmRLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
3594                 data = RLC_SERDES_WR_CTRL__BPM_ADDR_MASK |
3595                         RLC_SERDES_WR_CTRL__MGCG_OVERRIDE_0_MASK;
3596                 WREG32(mmRLC_SERDES_WR_CTRL, data);
3597                 mutex_unlock(&adev->grbm_idx_mutex);
3598
3599                 gfx_v7_0_update_rlc(adev, tmp);
3600
3601                 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGTS) {
3602                         orig = data = RREG32(mmCGTS_SM_CTRL_REG);
3603                         data &= ~CGTS_SM_CTRL_REG__SM_MODE_MASK;
3604                         data |= (0x2 << CGTS_SM_CTRL_REG__SM_MODE__SHIFT);
3605                         data |= CGTS_SM_CTRL_REG__SM_MODE_ENABLE_MASK;
3606                         data &= ~CGTS_SM_CTRL_REG__OVERRIDE_MASK;
3607                         if ((adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) &&
3608                             (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGTS_LS))
3609                                 data &= ~CGTS_SM_CTRL_REG__LS_OVERRIDE_MASK;
3610                         data &= ~CGTS_SM_CTRL_REG__ON_MONITOR_ADD_MASK;
3611                         data |= CGTS_SM_CTRL_REG__ON_MONITOR_ADD_EN_MASK;
3612                         data |= (0x96 << CGTS_SM_CTRL_REG__ON_MONITOR_ADD__SHIFT);
3613                         if (orig != data)
3614                                 WREG32(mmCGTS_SM_CTRL_REG, data);
3615                 }
3616         } else {
3617                 orig = data = RREG32(mmRLC_CGTT_MGCG_OVERRIDE);
3618                 data |= 0x00000003;
3619                 if (orig != data)
3620                         WREG32(mmRLC_CGTT_MGCG_OVERRIDE, data);
3621
3622                 data = RREG32(mmRLC_MEM_SLP_CNTL);
3623                 if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK) {
3624                         data &= ~RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK;
3625                         WREG32(mmRLC_MEM_SLP_CNTL, data);
3626                 }
3627
3628                 data = RREG32(mmCP_MEM_SLP_CNTL);
3629                 if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK) {
3630                         data &= ~CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK;
3631                         WREG32(mmCP_MEM_SLP_CNTL, data);
3632                 }
3633
3634                 orig = data = RREG32(mmCGTS_SM_CTRL_REG);
3635                 data |= CGTS_SM_CTRL_REG__OVERRIDE_MASK | CGTS_SM_CTRL_REG__LS_OVERRIDE_MASK;
3636                 if (orig != data)
3637                         WREG32(mmCGTS_SM_CTRL_REG, data);
3638
3639                 tmp = gfx_v7_0_halt_rlc(adev);
3640
3641                 mutex_lock(&adev->grbm_idx_mutex);
3642                 gfx_v7_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
3643                 WREG32(mmRLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
3644                 WREG32(mmRLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
3645                 data = RLC_SERDES_WR_CTRL__BPM_ADDR_MASK | RLC_SERDES_WR_CTRL__MGCG_OVERRIDE_1_MASK;
3646                 WREG32(mmRLC_SERDES_WR_CTRL, data);
3647                 mutex_unlock(&adev->grbm_idx_mutex);
3648
3649                 gfx_v7_0_update_rlc(adev, tmp);
3650         }
3651 }
3652
3653 static void gfx_v7_0_update_cg(struct amdgpu_device *adev,
3654                                bool enable)
3655 {
3656         gfx_v7_0_enable_gui_idle_interrupt(adev, false);
3657         /* order matters! */
3658         if (enable) {
3659                 gfx_v7_0_enable_mgcg(adev, true);
3660                 gfx_v7_0_enable_cgcg(adev, true);
3661         } else {
3662                 gfx_v7_0_enable_cgcg(adev, false);
3663                 gfx_v7_0_enable_mgcg(adev, false);
3664         }
3665         gfx_v7_0_enable_gui_idle_interrupt(adev, true);
3666 }
3667
3668 static void gfx_v7_0_enable_sclk_slowdown_on_pu(struct amdgpu_device *adev,
3669                                                 bool enable)
3670 {
3671         u32 data, orig;
3672
3673         orig = data = RREG32(mmRLC_PG_CNTL);
3674         if (enable && (adev->pg_flags & AMD_PG_SUPPORT_RLC_SMU_HS))
3675                 data |= RLC_PG_CNTL__SMU_CLK_SLOWDOWN_ON_PU_ENABLE_MASK;
3676         else
3677                 data &= ~RLC_PG_CNTL__SMU_CLK_SLOWDOWN_ON_PU_ENABLE_MASK;
3678         if (orig != data)
3679                 WREG32(mmRLC_PG_CNTL, data);
3680 }
3681
3682 static void gfx_v7_0_enable_sclk_slowdown_on_pd(struct amdgpu_device *adev,
3683                                                 bool enable)
3684 {
3685         u32 data, orig;
3686
3687         orig = data = RREG32(mmRLC_PG_CNTL);
3688         if (enable && (adev->pg_flags & AMD_PG_SUPPORT_RLC_SMU_HS))
3689                 data |= RLC_PG_CNTL__SMU_CLK_SLOWDOWN_ON_PD_ENABLE_MASK;
3690         else
3691                 data &= ~RLC_PG_CNTL__SMU_CLK_SLOWDOWN_ON_PD_ENABLE_MASK;
3692         if (orig != data)
3693                 WREG32(mmRLC_PG_CNTL, data);
3694 }
3695
3696 static void gfx_v7_0_enable_cp_pg(struct amdgpu_device *adev, bool enable)
3697 {
3698         u32 data, orig;
3699
3700         orig = data = RREG32(mmRLC_PG_CNTL);
3701         if (enable && (adev->pg_flags & AMD_PG_SUPPORT_CP))
3702                 data &= ~0x8000;
3703         else
3704                 data |= 0x8000;
3705         if (orig != data)
3706                 WREG32(mmRLC_PG_CNTL, data);
3707 }
3708
3709 static void gfx_v7_0_enable_gds_pg(struct amdgpu_device *adev, bool enable)
3710 {
3711         u32 data, orig;
3712
3713         orig = data = RREG32(mmRLC_PG_CNTL);
3714         if (enable && (adev->pg_flags & AMD_PG_SUPPORT_GDS))
3715                 data &= ~0x2000;
3716         else
3717                 data |= 0x2000;
3718         if (orig != data)
3719                 WREG32(mmRLC_PG_CNTL, data);
3720 }
3721
3722 static int gfx_v7_0_cp_pg_table_num(struct amdgpu_device *adev)
3723 {
3724         if (adev->asic_type == CHIP_KAVERI)
3725                 return 5;
3726         else
3727                 return 4;
3728 }
3729
3730 static void gfx_v7_0_enable_gfx_cgpg(struct amdgpu_device *adev,
3731                                      bool enable)
3732 {
3733         u32 data, orig;
3734
3735         if (enable && (adev->pg_flags & AMD_PG_SUPPORT_GFX_PG)) {
3736                 orig = data = RREG32(mmRLC_PG_CNTL);
3737                 data |= RLC_PG_CNTL__GFX_POWER_GATING_ENABLE_MASK;
3738                 if (orig != data)
3739                         WREG32(mmRLC_PG_CNTL, data);
3740
3741                 orig = data = RREG32(mmRLC_AUTO_PG_CTRL);
3742                 data |= RLC_AUTO_PG_CTRL__AUTO_PG_EN_MASK;
3743                 if (orig != data)
3744                         WREG32(mmRLC_AUTO_PG_CTRL, data);
3745         } else {
3746                 orig = data = RREG32(mmRLC_PG_CNTL);
3747                 data &= ~RLC_PG_CNTL__GFX_POWER_GATING_ENABLE_MASK;
3748                 if (orig != data)
3749                         WREG32(mmRLC_PG_CNTL, data);
3750
3751                 orig = data = RREG32(mmRLC_AUTO_PG_CTRL);
3752                 data &= ~RLC_AUTO_PG_CTRL__AUTO_PG_EN_MASK;
3753                 if (orig != data)
3754                         WREG32(mmRLC_AUTO_PG_CTRL, data);
3755
3756                 data = RREG32(mmDB_RENDER_CONTROL);
3757         }
3758 }
3759
3760 static void gfx_v7_0_set_user_cu_inactive_bitmap(struct amdgpu_device *adev,
3761                                                  u32 bitmap)
3762 {
3763         u32 data;
3764
3765         if (!bitmap)
3766                 return;
3767
3768         data = bitmap << GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT;
3769         data &= GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK;
3770
3771         WREG32(mmGC_USER_SHADER_ARRAY_CONFIG, data);
3772 }
3773
3774 static u32 gfx_v7_0_get_cu_active_bitmap(struct amdgpu_device *adev)
3775 {
3776         u32 data, mask;
3777
3778         data = RREG32(mmCC_GC_SHADER_ARRAY_CONFIG);
3779         data |= RREG32(mmGC_USER_SHADER_ARRAY_CONFIG);
3780
3781         data &= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK;
3782         data >>= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT;
3783
3784         mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_cu_per_sh);
3785
3786         return (~data) & mask;
3787 }
3788
3789 static void gfx_v7_0_init_ao_cu_mask(struct amdgpu_device *adev)
3790 {
3791         u32 tmp;
3792
3793         WREG32(mmRLC_PG_ALWAYS_ON_CU_MASK, adev->gfx.cu_info.ao_cu_mask);
3794
3795         tmp = RREG32(mmRLC_MAX_PG_CU);
3796         tmp &= ~RLC_MAX_PG_CU__MAX_POWERED_UP_CU_MASK;
3797         tmp |= (adev->gfx.cu_info.number << RLC_MAX_PG_CU__MAX_POWERED_UP_CU__SHIFT);
3798         WREG32(mmRLC_MAX_PG_CU, tmp);
3799 }
3800
3801 static void gfx_v7_0_enable_gfx_static_mgpg(struct amdgpu_device *adev,
3802                                             bool enable)
3803 {
3804         u32 data, orig;
3805
3806         orig = data = RREG32(mmRLC_PG_CNTL);
3807         if (enable && (adev->pg_flags & AMD_PG_SUPPORT_GFX_SMG))
3808                 data |= RLC_PG_CNTL__STATIC_PER_CU_PG_ENABLE_MASK;
3809         else
3810                 data &= ~RLC_PG_CNTL__STATIC_PER_CU_PG_ENABLE_MASK;
3811         if (orig != data)
3812                 WREG32(mmRLC_PG_CNTL, data);
3813 }
3814
3815 static void gfx_v7_0_enable_gfx_dynamic_mgpg(struct amdgpu_device *adev,
3816                                              bool enable)
3817 {
3818         u32 data, orig;
3819
3820         orig = data = RREG32(mmRLC_PG_CNTL);
3821         if (enable && (adev->pg_flags & AMD_PG_SUPPORT_GFX_DMG))
3822                 data |= RLC_PG_CNTL__DYN_PER_CU_PG_ENABLE_MASK;
3823         else
3824                 data &= ~RLC_PG_CNTL__DYN_PER_CU_PG_ENABLE_MASK;
3825         if (orig != data)
3826                 WREG32(mmRLC_PG_CNTL, data);
3827 }
3828
3829 #define RLC_SAVE_AND_RESTORE_STARTING_OFFSET 0x90
3830 #define RLC_CLEAR_STATE_DESCRIPTOR_OFFSET    0x3D
3831
3832 static void gfx_v7_0_init_gfx_cgpg(struct amdgpu_device *adev)
3833 {
3834         u32 data, orig;
3835         u32 i;
3836
3837         if (adev->gfx.rlc.cs_data) {
3838                 WREG32(mmRLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
3839                 WREG32(mmRLC_GPM_SCRATCH_DATA, upper_32_bits(adev->gfx.rlc.clear_state_gpu_addr));
3840                 WREG32(mmRLC_GPM_SCRATCH_DATA, lower_32_bits(adev->gfx.rlc.clear_state_gpu_addr));
3841                 WREG32(mmRLC_GPM_SCRATCH_DATA, adev->gfx.rlc.clear_state_size);
3842         } else {
3843                 WREG32(mmRLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
3844                 for (i = 0; i < 3; i++)
3845                         WREG32(mmRLC_GPM_SCRATCH_DATA, 0);
3846         }
3847         if (adev->gfx.rlc.reg_list) {
3848                 WREG32(mmRLC_GPM_SCRATCH_ADDR, RLC_SAVE_AND_RESTORE_STARTING_OFFSET);
3849                 for (i = 0; i < adev->gfx.rlc.reg_list_size; i++)
3850                         WREG32(mmRLC_GPM_SCRATCH_DATA, adev->gfx.rlc.reg_list[i]);
3851         }
3852
3853         orig = data = RREG32(mmRLC_PG_CNTL);
3854         data |= RLC_PG_CNTL__GFX_POWER_GATING_SRC_MASK;
3855         if (orig != data)
3856                 WREG32(mmRLC_PG_CNTL, data);
3857
3858         WREG32(mmRLC_SAVE_AND_RESTORE_BASE, adev->gfx.rlc.save_restore_gpu_addr >> 8);
3859         WREG32(mmRLC_JUMP_TABLE_RESTORE, adev->gfx.rlc.cp_table_gpu_addr >> 8);
3860
3861         data = RREG32(mmCP_RB_WPTR_POLL_CNTL);
3862         data &= ~CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK;
3863         data |= (0x60 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);
3864         WREG32(mmCP_RB_WPTR_POLL_CNTL, data);
3865
3866         data = 0x10101010;
3867         WREG32(mmRLC_PG_DELAY, data);
3868
3869         data = RREG32(mmRLC_PG_DELAY_2);
3870         data &= ~0xff;
3871         data |= 0x3;
3872         WREG32(mmRLC_PG_DELAY_2, data);
3873
3874         data = RREG32(mmRLC_AUTO_PG_CTRL);
3875         data &= ~RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD_MASK;
3876         data |= (0x700 << RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD__SHIFT);
3877         WREG32(mmRLC_AUTO_PG_CTRL, data);
3878
3879 }
3880
3881 static void gfx_v7_0_update_gfx_pg(struct amdgpu_device *adev, bool enable)
3882 {
3883         gfx_v7_0_enable_gfx_cgpg(adev, enable);
3884         gfx_v7_0_enable_gfx_static_mgpg(adev, enable);
3885         gfx_v7_0_enable_gfx_dynamic_mgpg(adev, enable);
3886 }
3887
3888 static u32 gfx_v7_0_get_csb_size(struct amdgpu_device *adev)
3889 {
3890         u32 count = 0;
3891         const struct cs_section_def *sect = NULL;
3892         const struct cs_extent_def *ext = NULL;
3893
3894         if (adev->gfx.rlc.cs_data == NULL)
3895                 return 0;
3896
3897         /* begin clear state */
3898         count += 2;
3899         /* context control state */
3900         count += 3;
3901
3902         for (sect = adev->gfx.rlc.cs_data; sect->section != NULL; ++sect) {
3903                 for (ext = sect->section; ext->extent != NULL; ++ext) {
3904                         if (sect->id == SECT_CONTEXT)
3905                                 count += 2 + ext->reg_count;
3906                         else
3907                                 return 0;
3908                 }
3909         }
3910         /* pa_sc_raster_config/pa_sc_raster_config1 */
3911         count += 4;
3912         /* end clear state */
3913         count += 2;
3914         /* clear state */
3915         count += 2;
3916
3917         return count;
3918 }
3919
3920 static void gfx_v7_0_get_csb_buffer(struct amdgpu_device *adev,
3921                                     volatile u32 *buffer)
3922 {
3923         u32 count = 0, i;
3924         const struct cs_section_def *sect = NULL;
3925         const struct cs_extent_def *ext = NULL;
3926
3927         if (adev->gfx.rlc.cs_data == NULL)
3928                 return;
3929         if (buffer == NULL)
3930                 return;
3931
3932         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3933         buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3934
3935         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
3936         buffer[count++] = cpu_to_le32(0x80000000);
3937         buffer[count++] = cpu_to_le32(0x80000000);
3938
3939         for (sect = adev->gfx.rlc.cs_data; sect->section != NULL; ++sect) {
3940                 for (ext = sect->section; ext->extent != NULL; ++ext) {
3941                         if (sect->id == SECT_CONTEXT) {
3942                                 buffer[count++] =
3943                                         cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
3944                                 buffer[count++] = cpu_to_le32(ext->reg_index - PACKET3_SET_CONTEXT_REG_START);
3945                                 for (i = 0; i < ext->reg_count; i++)
3946                                         buffer[count++] = cpu_to_le32(ext->extent[i]);
3947                         } else {
3948                                 return;
3949                         }
3950                 }
3951         }
3952
3953         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 2));
3954         buffer[count++] = cpu_to_le32(mmPA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
3955         switch (adev->asic_type) {
3956         case CHIP_BONAIRE:
3957                 buffer[count++] = cpu_to_le32(0x16000012);
3958                 buffer[count++] = cpu_to_le32(0x00000000);
3959                 break;
3960         case CHIP_KAVERI:
3961                 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
3962                 buffer[count++] = cpu_to_le32(0x00000000);
3963                 break;
3964         case CHIP_KABINI:
3965         case CHIP_MULLINS:
3966                 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
3967                 buffer[count++] = cpu_to_le32(0x00000000);
3968                 break;
3969         case CHIP_HAWAII:
3970                 buffer[count++] = cpu_to_le32(0x3a00161a);
3971                 buffer[count++] = cpu_to_le32(0x0000002e);
3972                 break;
3973         default:
3974                 buffer[count++] = cpu_to_le32(0x00000000);
3975                 buffer[count++] = cpu_to_le32(0x00000000);
3976                 break;
3977         }
3978
3979         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3980         buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
3981
3982         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
3983         buffer[count++] = cpu_to_le32(0);
3984 }
3985
3986 static void gfx_v7_0_init_pg(struct amdgpu_device *adev)
3987 {
3988         if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG |
3989                               AMD_PG_SUPPORT_GFX_SMG |
3990                               AMD_PG_SUPPORT_GFX_DMG |
3991                               AMD_PG_SUPPORT_CP |
3992                               AMD_PG_SUPPORT_GDS |
3993                               AMD_PG_SUPPORT_RLC_SMU_HS)) {
3994                 gfx_v7_0_enable_sclk_slowdown_on_pu(adev, true);
3995                 gfx_v7_0_enable_sclk_slowdown_on_pd(adev, true);
3996                 if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PG) {
3997                         gfx_v7_0_init_gfx_cgpg(adev);
3998                         gfx_v7_0_enable_cp_pg(adev, true);
3999                         gfx_v7_0_enable_gds_pg(adev, true);
4000                 }
4001                 gfx_v7_0_init_ao_cu_mask(adev);
4002                 gfx_v7_0_update_gfx_pg(adev, true);
4003         }
4004 }
4005
4006 static void gfx_v7_0_fini_pg(struct amdgpu_device *adev)
4007 {
4008         if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG |
4009                               AMD_PG_SUPPORT_GFX_SMG |
4010                               AMD_PG_SUPPORT_GFX_DMG |
4011                               AMD_PG_SUPPORT_CP |
4012                               AMD_PG_SUPPORT_GDS |
4013                               AMD_PG_SUPPORT_RLC_SMU_HS)) {
4014                 gfx_v7_0_update_gfx_pg(adev, false);
4015                 if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PG) {
4016                         gfx_v7_0_enable_cp_pg(adev, false);
4017                         gfx_v7_0_enable_gds_pg(adev, false);
4018                 }
4019         }
4020 }
4021
4022 /**
4023  * gfx_v7_0_get_gpu_clock_counter - return GPU clock counter snapshot
4024  *
4025  * @adev: amdgpu_device pointer
4026  *
4027  * Fetches a GPU clock counter snapshot (SI).
4028  * Returns the 64 bit clock counter snapshot.
4029  */
4030 static uint64_t gfx_v7_0_get_gpu_clock_counter(struct amdgpu_device *adev)
4031 {
4032         uint64_t clock;
4033
4034         mutex_lock(&adev->gfx.gpu_clock_mutex);
4035         WREG32(mmRLC_CAPTURE_GPU_CLOCK_COUNT, 1);
4036         clock = (uint64_t)RREG32(mmRLC_GPU_CLOCK_COUNT_LSB) |
4037                 ((uint64_t)RREG32(mmRLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
4038         mutex_unlock(&adev->gfx.gpu_clock_mutex);
4039         return clock;
4040 }
4041
4042 static void gfx_v7_0_ring_emit_gds_switch(struct amdgpu_ring *ring,
4043                                           uint32_t vmid,
4044                                           uint32_t gds_base, uint32_t gds_size,
4045                                           uint32_t gws_base, uint32_t gws_size,
4046                                           uint32_t oa_base, uint32_t oa_size)
4047 {
4048         /* GDS Base */
4049         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4050         amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4051                                 WRITE_DATA_DST_SEL(0)));
4052         amdgpu_ring_write(ring, amdgpu_gds_reg_offset[vmid].mem_base);
4053         amdgpu_ring_write(ring, 0);
4054         amdgpu_ring_write(ring, gds_base);
4055
4056         /* GDS Size */
4057         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4058         amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4059                                 WRITE_DATA_DST_SEL(0)));
4060         amdgpu_ring_write(ring, amdgpu_gds_reg_offset[vmid].mem_size);
4061         amdgpu_ring_write(ring, 0);
4062         amdgpu_ring_write(ring, gds_size);
4063
4064         /* GWS */
4065         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4066         amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4067                                 WRITE_DATA_DST_SEL(0)));
4068         amdgpu_ring_write(ring, amdgpu_gds_reg_offset[vmid].gws);
4069         amdgpu_ring_write(ring, 0);
4070         amdgpu_ring_write(ring, gws_size << GDS_GWS_VMID0__SIZE__SHIFT | gws_base);
4071
4072         /* OA */
4073         amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4074         amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4075                                 WRITE_DATA_DST_SEL(0)));
4076         amdgpu_ring_write(ring, amdgpu_gds_reg_offset[vmid].oa);
4077         amdgpu_ring_write(ring, 0);
4078         amdgpu_ring_write(ring, (1 << (oa_size + oa_base)) - (1 << oa_base));
4079 }
4080
4081 static void gfx_v7_0_ring_soft_recovery(struct amdgpu_ring *ring, unsigned vmid)
4082 {
4083         struct amdgpu_device *adev = ring->adev;
4084         uint32_t value = 0;
4085
4086         value = REG_SET_FIELD(value, SQ_CMD, CMD, 0x03);
4087         value = REG_SET_FIELD(value, SQ_CMD, MODE, 0x01);
4088         value = REG_SET_FIELD(value, SQ_CMD, CHECK_VMID, 1);
4089         value = REG_SET_FIELD(value, SQ_CMD, VM_ID, vmid);
4090         WREG32(mmSQ_CMD, value);
4091 }
4092
4093 static uint32_t wave_read_ind(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t address)
4094 {
4095         WREG32(mmSQ_IND_INDEX,
4096                 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
4097                 (simd << SQ_IND_INDEX__SIMD_ID__SHIFT) |
4098                 (address << SQ_IND_INDEX__INDEX__SHIFT) |
4099                 (SQ_IND_INDEX__FORCE_READ_MASK));
4100         return RREG32(mmSQ_IND_DATA);
4101 }
4102
4103 static void wave_read_regs(struct amdgpu_device *adev, uint32_t simd,
4104                            uint32_t wave, uint32_t thread,
4105                            uint32_t regno, uint32_t num, uint32_t *out)
4106 {
4107         WREG32(mmSQ_IND_INDEX,
4108                 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
4109                 (simd << SQ_IND_INDEX__SIMD_ID__SHIFT) |
4110                 (regno << SQ_IND_INDEX__INDEX__SHIFT) |
4111                 (thread << SQ_IND_INDEX__THREAD_ID__SHIFT) |
4112                 (SQ_IND_INDEX__FORCE_READ_MASK) |
4113                 (SQ_IND_INDEX__AUTO_INCR_MASK));
4114         while (num--)
4115                 *(out++) = RREG32(mmSQ_IND_DATA);
4116 }
4117
4118 static void gfx_v7_0_read_wave_data(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t *dst, int *no_fields)
4119 {
4120         /* type 0 wave data */
4121         dst[(*no_fields)++] = 0;
4122         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_STATUS);
4123         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_LO);
4124         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_HI);
4125         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_LO);
4126         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_HI);
4127         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_HW_ID);
4128         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW0);
4129         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW1);
4130         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_GPR_ALLOC);
4131         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_LDS_ALLOC);
4132         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TRAPSTS);
4133         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_STS);
4134         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TBA_LO);
4135         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TBA_HI);
4136         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TMA_LO);
4137         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TMA_HI);
4138         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_DBG0);
4139         dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_M0);
4140 }
4141
4142 static void gfx_v7_0_read_wave_sgprs(struct amdgpu_device *adev, uint32_t simd,
4143                                      uint32_t wave, uint32_t start,
4144                                      uint32_t size, uint32_t *dst)
4145 {
4146         wave_read_regs(
4147                 adev, simd, wave, 0,
4148                 start + SQIND_WAVE_SGPRS_OFFSET, size, dst);
4149 }
4150
4151 static void gfx_v7_0_select_me_pipe_q(struct amdgpu_device *adev,
4152                                   u32 me, u32 pipe, u32 q, u32 vm)
4153 {
4154         cik_srbm_select(adev, me, pipe, q, vm);
4155 }
4156
4157 static const struct amdgpu_gfx_funcs gfx_v7_0_gfx_funcs = {
4158         .get_gpu_clock_counter = &gfx_v7_0_get_gpu_clock_counter,
4159         .select_se_sh = &gfx_v7_0_select_se_sh,
4160         .read_wave_data = &gfx_v7_0_read_wave_data,
4161         .read_wave_sgprs = &gfx_v7_0_read_wave_sgprs,
4162         .select_me_pipe_q = &gfx_v7_0_select_me_pipe_q
4163 };
4164
4165 static const struct amdgpu_rlc_funcs gfx_v7_0_rlc_funcs = {
4166         .is_rlc_enabled = gfx_v7_0_is_rlc_enabled,
4167         .set_safe_mode = gfx_v7_0_set_safe_mode,
4168         .unset_safe_mode = gfx_v7_0_unset_safe_mode,
4169         .init = gfx_v7_0_rlc_init,
4170         .get_csb_size = gfx_v7_0_get_csb_size,
4171         .get_csb_buffer = gfx_v7_0_get_csb_buffer,
4172         .get_cp_table_num = gfx_v7_0_cp_pg_table_num,
4173         .resume = gfx_v7_0_rlc_resume,
4174         .stop = gfx_v7_0_rlc_stop,
4175         .reset = gfx_v7_0_rlc_reset,
4176         .start = gfx_v7_0_rlc_start
4177 };
4178
4179 static int gfx_v7_0_early_init(void *handle)
4180 {
4181         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4182
4183         adev->gfx.num_gfx_rings = GFX7_NUM_GFX_RINGS;
4184         adev->gfx.num_compute_rings = AMDGPU_MAX_COMPUTE_RINGS;
4185         adev->gfx.funcs = &gfx_v7_0_gfx_funcs;
4186         adev->gfx.rlc.funcs = &gfx_v7_0_rlc_funcs;
4187         gfx_v7_0_set_ring_funcs(adev);
4188         gfx_v7_0_set_irq_funcs(adev);
4189         gfx_v7_0_set_gds_init(adev);
4190
4191         return 0;
4192 }
4193
4194 static int gfx_v7_0_late_init(void *handle)
4195 {
4196         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4197         int r;
4198
4199         r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0);
4200         if (r)
4201                 return r;
4202
4203         r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0);
4204         if (r)
4205                 return r;
4206
4207         return 0;
4208 }
4209
4210 static void gfx_v7_0_gpu_early_init(struct amdgpu_device *adev)
4211 {
4212         u32 gb_addr_config;
4213         u32 mc_shared_chmap, mc_arb_ramcfg;
4214         u32 dimm00_addr_map, dimm01_addr_map, dimm10_addr_map, dimm11_addr_map;
4215         u32 tmp;
4216
4217         switch (adev->asic_type) {
4218         case CHIP_BONAIRE:
4219                 adev->gfx.config.max_shader_engines = 2;
4220                 adev->gfx.config.max_tile_pipes = 4;
4221                 adev->gfx.config.max_cu_per_sh = 7;
4222                 adev->gfx.config.max_sh_per_se = 1;
4223                 adev->gfx.config.max_backends_per_se = 2;
4224                 adev->gfx.config.max_texture_channel_caches = 4;
4225                 adev->gfx.config.max_gprs = 256;
4226                 adev->gfx.config.max_gs_threads = 32;
4227                 adev->gfx.config.max_hw_contexts = 8;
4228
4229                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
4230                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
4231                 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
4232                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130;
4233                 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
4234                 break;
4235         case CHIP_HAWAII:
4236                 adev->gfx.config.max_shader_engines = 4;
4237                 adev->gfx.config.max_tile_pipes = 16;
4238                 adev->gfx.config.max_cu_per_sh = 11;
4239                 adev->gfx.config.max_sh_per_se = 1;
4240                 adev->gfx.config.max_backends_per_se = 4;
4241                 adev->gfx.config.max_texture_channel_caches = 16;
4242                 adev->gfx.config.max_gprs = 256;
4243                 adev->gfx.config.max_gs_threads = 32;
4244                 adev->gfx.config.max_hw_contexts = 8;
4245
4246                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
4247                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
4248                 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
4249                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130;
4250                 gb_addr_config = HAWAII_GB_ADDR_CONFIG_GOLDEN;
4251                 break;
4252         case CHIP_KAVERI:
4253                 adev->gfx.config.max_shader_engines = 1;
4254                 adev->gfx.config.max_tile_pipes = 4;
4255                 adev->gfx.config.max_cu_per_sh = 8;
4256                 adev->gfx.config.max_backends_per_se = 2;
4257                 adev->gfx.config.max_sh_per_se = 1;
4258                 adev->gfx.config.max_texture_channel_caches = 4;
4259                 adev->gfx.config.max_gprs = 256;
4260                 adev->gfx.config.max_gs_threads = 16;
4261                 adev->gfx.config.max_hw_contexts = 8;
4262
4263                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
4264                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
4265                 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
4266                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130;
4267                 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
4268                 break;
4269         case CHIP_KABINI:
4270         case CHIP_MULLINS:
4271         default:
4272                 adev->gfx.config.max_shader_engines = 1;
4273                 adev->gfx.config.max_tile_pipes = 2;
4274                 adev->gfx.config.max_cu_per_sh = 2;
4275                 adev->gfx.config.max_sh_per_se = 1;
4276                 adev->gfx.config.max_backends_per_se = 1;
4277                 adev->gfx.config.max_texture_channel_caches = 2;
4278                 adev->gfx.config.max_gprs = 256;
4279                 adev->gfx.config.max_gs_threads = 16;
4280                 adev->gfx.config.max_hw_contexts = 8;
4281
4282                 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
4283                 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
4284                 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
4285                 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130;
4286                 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
4287                 break;
4288         }
4289
4290         mc_shared_chmap = RREG32(mmMC_SHARED_CHMAP);
4291         adev->gfx.config.mc_arb_ramcfg = RREG32(mmMC_ARB_RAMCFG);
4292         mc_arb_ramcfg = adev->gfx.config.mc_arb_ramcfg;
4293
4294         adev->gfx.config.num_tile_pipes = adev->gfx.config.max_tile_pipes;
4295         adev->gfx.config.mem_max_burst_length_bytes = 256;
4296         if (adev->flags & AMD_IS_APU) {
4297                 /* Get memory bank mapping mode. */
4298                 tmp = RREG32(mmMC_FUS_DRAM0_BANK_ADDR_MAPPING);
4299                 dimm00_addr_map = REG_GET_FIELD(tmp, MC_FUS_DRAM0_BANK_ADDR_MAPPING, DIMM0ADDRMAP);
4300                 dimm01_addr_map = REG_GET_FIELD(tmp, MC_FUS_DRAM0_BANK_ADDR_MAPPING, DIMM1ADDRMAP);
4301
4302                 tmp = RREG32(mmMC_FUS_DRAM1_BANK_ADDR_MAPPING);
4303                 dimm10_addr_map = REG_GET_FIELD(tmp, MC_FUS_DRAM1_BANK_ADDR_MAPPING, DIMM0ADDRMAP);
4304                 dimm11_addr_map = REG_GET_FIELD(tmp, MC_FUS_DRAM1_BANK_ADDR_MAPPING, DIMM1ADDRMAP);
4305
4306                 /* Validate settings in case only one DIMM installed. */
4307                 if ((dimm00_addr_map == 0) || (dimm00_addr_map == 3) || (dimm00_addr_map == 4) || (dimm00_addr_map > 12))
4308                         dimm00_addr_map = 0;
4309                 if ((dimm01_addr_map == 0) || (dimm01_addr_map == 3) || (dimm01_addr_map == 4) || (dimm01_addr_map > 12))
4310                         dimm01_addr_map = 0;
4311                 if ((dimm10_addr_map == 0) || (dimm10_addr_map == 3) || (dimm10_addr_map == 4) || (dimm10_addr_map > 12))
4312                         dimm10_addr_map = 0;
4313                 if ((dimm11_addr_map == 0) || (dimm11_addr_map == 3) || (dimm11_addr_map == 4) || (dimm11_addr_map > 12))
4314                         dimm11_addr_map = 0;
4315
4316                 /* If DIMM Addr map is 8GB, ROW size should be 2KB. Otherwise 1KB. */
4317                 /* If ROW size(DIMM1) != ROW size(DMIMM0), ROW size should be larger one. */
4318                 if ((dimm00_addr_map == 11) || (dimm01_addr_map == 11) || (dimm10_addr_map == 11) || (dimm11_addr_map == 11))
4319                         adev->gfx.config.mem_row_size_in_kb = 2;
4320                 else
4321                         adev->gfx.config.mem_row_size_in_kb = 1;
4322         } else {
4323                 tmp = (mc_arb_ramcfg & MC_ARB_RAMCFG__NOOFCOLS_MASK) >> MC_ARB_RAMCFG__NOOFCOLS__SHIFT;
4324                 adev->gfx.config.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
4325                 if (adev->gfx.config.mem_row_size_in_kb > 4)
4326                         adev->gfx.config.mem_row_size_in_kb = 4;
4327         }
4328         /* XXX use MC settings? */
4329         adev->gfx.config.shader_engine_tile_size = 32;
4330         adev->gfx.config.num_gpus = 1;
4331         adev->gfx.config.multi_gpu_tile_size = 64;
4332
4333         /* fix up row size */
4334         gb_addr_config &= ~GB_ADDR_CONFIG__ROW_SIZE_MASK;
4335         switch (adev->gfx.config.mem_row_size_in_kb) {
4336         case 1:
4337         default:
4338                 gb_addr_config |= (0 << GB_ADDR_CONFIG__ROW_SIZE__SHIFT);
4339                 break;
4340         case 2:
4341                 gb_addr_config |= (1 << GB_ADDR_CONFIG__ROW_SIZE__SHIFT);
4342                 break;
4343         case 4:
4344                 gb_addr_config |= (2 << GB_ADDR_CONFIG__ROW_SIZE__SHIFT);
4345                 break;
4346         }
4347         adev->gfx.config.gb_addr_config = gb_addr_config;
4348 }
4349
4350 static int gfx_v7_0_compute_ring_init(struct amdgpu_device *adev, int ring_id,
4351                                         int mec, int pipe, int queue)
4352 {
4353         int r;
4354         unsigned irq_type;
4355         struct amdgpu_ring *ring = &adev->gfx.compute_ring[ring_id];
4356
4357         /* mec0 is me1 */
4358         ring->me = mec + 1;
4359         ring->pipe = pipe;
4360         ring->queue = queue;
4361
4362         ring->ring_obj = NULL;
4363         ring->use_doorbell = true;
4364         ring->doorbell_index = adev->doorbell_index.mec_ring0 + ring_id;
4365         sprintf(ring->name, "comp_%d.%d.%d", ring->me, ring->pipe, ring->queue);
4366
4367         irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP
4368                 + ((ring->me - 1) * adev->gfx.mec.num_pipe_per_mec)
4369                 + ring->pipe;
4370
4371         /* type-2 packets are deprecated on MEC, use type-3 instead */
4372         r = amdgpu_ring_init(adev, ring, 1024,
4373                         &adev->gfx.eop_irq, irq_type);
4374         if (r)
4375                 return r;
4376
4377
4378         return 0;
4379 }
4380
4381 static int gfx_v7_0_sw_init(void *handle)
4382 {
4383         struct amdgpu_ring *ring;
4384         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4385         int i, j, k, r, ring_id;
4386
4387         switch (adev->asic_type) {
4388         case CHIP_KAVERI:
4389                 adev->gfx.mec.num_mec = 2;
4390                 break;
4391         case CHIP_BONAIRE:
4392         case CHIP_HAWAII:
4393         case CHIP_KABINI:
4394         case CHIP_MULLINS:
4395         default:
4396                 adev->gfx.mec.num_mec = 1;
4397                 break;
4398         }
4399         adev->gfx.mec.num_pipe_per_mec = 4;
4400         adev->gfx.mec.num_queue_per_pipe = 8;
4401
4402         /* EOP Event */
4403         r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, 181, &adev->gfx.eop_irq);
4404         if (r)
4405                 return r;
4406
4407         /* Privileged reg */
4408         r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, 184,
4409                               &adev->gfx.priv_reg_irq);
4410         if (r)
4411                 return r;
4412
4413         /* Privileged inst */
4414         r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, 185,
4415                               &adev->gfx.priv_inst_irq);
4416         if (r)
4417                 return r;
4418
4419         gfx_v7_0_scratch_init(adev);
4420
4421         r = gfx_v7_0_init_microcode(adev);
4422         if (r) {
4423                 DRM_ERROR("Failed to load gfx firmware!\n");
4424                 return r;
4425         }
4426
4427         r = adev->gfx.rlc.funcs->init(adev);
4428         if (r) {
4429                 DRM_ERROR("Failed to init rlc BOs!\n");
4430                 return r;
4431         }
4432
4433         /* allocate mec buffers */
4434         r = gfx_v7_0_mec_init(adev);
4435         if (r) {
4436                 DRM_ERROR("Failed to init MEC BOs!\n");
4437                 return r;
4438         }
4439
4440         for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
4441                 ring = &adev->gfx.gfx_ring[i];
4442                 ring->ring_obj = NULL;
4443                 sprintf(ring->name, "gfx");
4444                 r = amdgpu_ring_init(adev, ring, 1024,
4445                                      &adev->gfx.eop_irq, AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP);
4446                 if (r)
4447                         return r;
4448         }
4449
4450         /* set up the compute queues - allocate horizontally across pipes */
4451         ring_id = 0;
4452         for (i = 0; i < adev->gfx.mec.num_mec; ++i) {
4453                 for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) {
4454                         for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) {
4455                                 if (!amdgpu_gfx_is_mec_queue_enabled(adev, i, k, j))
4456                                         continue;
4457
4458                                 r = gfx_v7_0_compute_ring_init(adev,
4459                                                                 ring_id,
4460                                                                 i, k, j);
4461                                 if (r)
4462                                         return r;
4463
4464                                 ring_id++;
4465                         }
4466                 }
4467         }
4468
4469         adev->gfx.ce_ram_size = 0x8000;
4470
4471         gfx_v7_0_gpu_early_init(adev);
4472
4473         return r;
4474 }
4475
4476 static int gfx_v7_0_sw_fini(void *handle)
4477 {
4478         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4479         int i;
4480
4481         for (i = 0; i < adev->gfx.num_gfx_rings; i++)
4482                 amdgpu_ring_fini(&adev->gfx.gfx_ring[i]);
4483         for (i = 0; i < adev->gfx.num_compute_rings; i++)
4484                 amdgpu_ring_fini(&adev->gfx.compute_ring[i]);
4485
4486         gfx_v7_0_cp_compute_fini(adev);
4487         amdgpu_gfx_rlc_fini(adev);
4488         gfx_v7_0_mec_fini(adev);
4489         amdgpu_bo_free_kernel(&adev->gfx.rlc.clear_state_obj,
4490                                 &adev->gfx.rlc.clear_state_gpu_addr,
4491                                 (void **)&adev->gfx.rlc.cs_ptr);
4492         if (adev->gfx.rlc.cp_table_size) {
4493                 amdgpu_bo_free_kernel(&adev->gfx.rlc.cp_table_obj,
4494                                 &adev->gfx.rlc.cp_table_gpu_addr,
4495                                 (void **)&adev->gfx.rlc.cp_table_ptr);
4496         }
4497         gfx_v7_0_free_microcode(adev);
4498
4499         return 0;
4500 }
4501
4502 static int gfx_v7_0_hw_init(void *handle)
4503 {
4504         int r;
4505         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4506
4507         gfx_v7_0_constants_init(adev);
4508
4509         /* init rlc */
4510         r = adev->gfx.rlc.funcs->resume(adev);
4511         if (r)
4512                 return r;
4513
4514         r = gfx_v7_0_cp_resume(adev);
4515         if (r)
4516                 return r;
4517
4518         return r;
4519 }
4520
4521 static int gfx_v7_0_hw_fini(void *handle)
4522 {
4523         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4524
4525         amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0);
4526         amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0);
4527         gfx_v7_0_cp_enable(adev, false);
4528         adev->gfx.rlc.funcs->stop(adev);
4529         gfx_v7_0_fini_pg(adev);
4530
4531         return 0;
4532 }
4533
4534 static int gfx_v7_0_suspend(void *handle)
4535 {
4536         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4537
4538         return gfx_v7_0_hw_fini(adev);
4539 }
4540
4541 static int gfx_v7_0_resume(void *handle)
4542 {
4543         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4544
4545         return gfx_v7_0_hw_init(adev);
4546 }
4547
4548 static bool gfx_v7_0_is_idle(void *handle)
4549 {
4550         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4551
4552         if (RREG32(mmGRBM_STATUS) & GRBM_STATUS__GUI_ACTIVE_MASK)
4553                 return false;
4554         else
4555                 return true;
4556 }
4557
4558 static int gfx_v7_0_wait_for_idle(void *handle)
4559 {
4560         unsigned i;
4561         u32 tmp;
4562         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4563
4564         for (i = 0; i < adev->usec_timeout; i++) {
4565                 /* read MC_STATUS */
4566                 tmp = RREG32(mmGRBM_STATUS) & GRBM_STATUS__GUI_ACTIVE_MASK;
4567
4568                 if (!tmp)
4569                         return 0;
4570                 udelay(1);
4571         }
4572         return -ETIMEDOUT;
4573 }
4574
4575 static int gfx_v7_0_soft_reset(void *handle)
4576 {
4577         u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
4578         u32 tmp;
4579         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4580
4581         /* GRBM_STATUS */
4582         tmp = RREG32(mmGRBM_STATUS);
4583         if (tmp & (GRBM_STATUS__PA_BUSY_MASK | GRBM_STATUS__SC_BUSY_MASK |
4584                    GRBM_STATUS__BCI_BUSY_MASK | GRBM_STATUS__SX_BUSY_MASK |
4585                    GRBM_STATUS__TA_BUSY_MASK | GRBM_STATUS__VGT_BUSY_MASK |
4586                    GRBM_STATUS__DB_BUSY_MASK | GRBM_STATUS__CB_BUSY_MASK |
4587                    GRBM_STATUS__GDS_BUSY_MASK | GRBM_STATUS__SPI_BUSY_MASK |
4588                    GRBM_STATUS__IA_BUSY_MASK | GRBM_STATUS__IA_BUSY_NO_DMA_MASK))
4589                 grbm_soft_reset |= GRBM_SOFT_RESET__SOFT_RESET_CP_MASK |
4590                         GRBM_SOFT_RESET__SOFT_RESET_GFX_MASK;
4591
4592         if (tmp & (GRBM_STATUS__CP_BUSY_MASK | GRBM_STATUS__CP_COHERENCY_BUSY_MASK)) {
4593                 grbm_soft_reset |= GRBM_SOFT_RESET__SOFT_RESET_CP_MASK;
4594                 srbm_soft_reset |= SRBM_SOFT_RESET__SOFT_RESET_GRBM_MASK;
4595         }
4596
4597         /* GRBM_STATUS2 */
4598         tmp = RREG32(mmGRBM_STATUS2);
4599         if (tmp & GRBM_STATUS2__RLC_BUSY_MASK)
4600                 grbm_soft_reset |= GRBM_SOFT_RESET__SOFT_RESET_RLC_MASK;
4601
4602         /* SRBM_STATUS */
4603         tmp = RREG32(mmSRBM_STATUS);
4604         if (tmp & SRBM_STATUS__GRBM_RQ_PENDING_MASK)
4605                 srbm_soft_reset |= SRBM_SOFT_RESET__SOFT_RESET_GRBM_MASK;
4606
4607         if (grbm_soft_reset || srbm_soft_reset) {
4608                 /* disable CG/PG */
4609                 gfx_v7_0_fini_pg(adev);
4610                 gfx_v7_0_update_cg(adev, false);
4611
4612                 /* stop the rlc */
4613                 adev->gfx.rlc.funcs->stop(adev);
4614
4615                 /* Disable GFX parsing/prefetching */
4616                 WREG32(mmCP_ME_CNTL, CP_ME_CNTL__ME_HALT_MASK | CP_ME_CNTL__PFP_HALT_MASK | CP_ME_CNTL__CE_HALT_MASK);
4617
4618                 /* Disable MEC parsing/prefetching */
4619                 WREG32(mmCP_MEC_CNTL, CP_MEC_CNTL__MEC_ME1_HALT_MASK | CP_MEC_CNTL__MEC_ME2_HALT_MASK);
4620
4621                 if (grbm_soft_reset) {
4622                         tmp = RREG32(mmGRBM_SOFT_RESET);
4623                         tmp |= grbm_soft_reset;
4624                         dev_info(adev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
4625                         WREG32(mmGRBM_SOFT_RESET, tmp);
4626                         tmp = RREG32(mmGRBM_SOFT_RESET);
4627
4628                         udelay(50);
4629
4630                         tmp &= ~grbm_soft_reset;
4631                         WREG32(mmGRBM_SOFT_RESET, tmp);
4632                         tmp = RREG32(mmGRBM_SOFT_RESET);
4633                 }
4634
4635                 if (srbm_soft_reset) {
4636                         tmp = RREG32(mmSRBM_SOFT_RESET);
4637                         tmp |= srbm_soft_reset;
4638                         dev_info(adev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
4639                         WREG32(mmSRBM_SOFT_RESET, tmp);
4640                         tmp = RREG32(mmSRBM_SOFT_RESET);
4641
4642                         udelay(50);
4643
4644                         tmp &= ~srbm_soft_reset;
4645                         WREG32(mmSRBM_SOFT_RESET, tmp);
4646                         tmp = RREG32(mmSRBM_SOFT_RESET);
4647                 }
4648                 /* Wait a little for things to settle down */
4649                 udelay(50);
4650         }
4651         return 0;
4652 }
4653
4654 static void gfx_v7_0_set_gfx_eop_interrupt_state(struct amdgpu_device *adev,
4655                                                  enum amdgpu_interrupt_state state)
4656 {
4657         u32 cp_int_cntl;
4658
4659         switch (state) {
4660         case AMDGPU_IRQ_STATE_DISABLE:
4661                 cp_int_cntl = RREG32(mmCP_INT_CNTL_RING0);
4662                 cp_int_cntl &= ~CP_INT_CNTL_RING0__TIME_STAMP_INT_ENABLE_MASK;
4663                 WREG32(mmCP_INT_CNTL_RING0, cp_int_cntl);
4664                 break;
4665         case AMDGPU_IRQ_STATE_ENABLE:
4666                 cp_int_cntl = RREG32(mmCP_INT_CNTL_RING0);
4667                 cp_int_cntl |= CP_INT_CNTL_RING0__TIME_STAMP_INT_ENABLE_MASK;
4668                 WREG32(mmCP_INT_CNTL_RING0, cp_int_cntl);
4669                 break;
4670         default:
4671                 break;
4672         }
4673 }
4674
4675 static void gfx_v7_0_set_compute_eop_interrupt_state(struct amdgpu_device *adev,
4676                                                      int me, int pipe,
4677                                                      enum amdgpu_interrupt_state state)
4678 {
4679         u32 mec_int_cntl, mec_int_cntl_reg;
4680
4681         /*
4682          * amdgpu controls only the first MEC. That's why this function only
4683          * handles the setting of interrupts for this specific MEC. All other
4684          * pipes' interrupts are set by amdkfd.
4685          */
4686
4687         if (me == 1) {
4688                 switch (pipe) {
4689                 case 0:
4690                         mec_int_cntl_reg = mmCP_ME1_PIPE0_INT_CNTL;
4691                         break;
4692                 case 1:
4693                         mec_int_cntl_reg = mmCP_ME1_PIPE1_INT_CNTL;
4694                         break;
4695                 case 2:
4696                         mec_int_cntl_reg = mmCP_ME1_PIPE2_INT_CNTL;
4697                         break;
4698                 case 3:
4699                         mec_int_cntl_reg = mmCP_ME1_PIPE3_INT_CNTL;
4700                         break;
4701                 default:
4702                         DRM_DEBUG("invalid pipe %d\n", pipe);
4703                         return;
4704                 }
4705         } else {
4706                 DRM_DEBUG("invalid me %d\n", me);
4707                 return;
4708         }
4709
4710         switch (state) {
4711         case AMDGPU_IRQ_STATE_DISABLE:
4712                 mec_int_cntl = RREG32(mec_int_cntl_reg);
4713                 mec_int_cntl &= ~CP_INT_CNTL_RING0__TIME_STAMP_INT_ENABLE_MASK;
4714                 WREG32(mec_int_cntl_reg, mec_int_cntl);
4715                 break;
4716         case AMDGPU_IRQ_STATE_ENABLE:
4717                 mec_int_cntl = RREG32(mec_int_cntl_reg);
4718                 mec_int_cntl |= CP_INT_CNTL_RING0__TIME_STAMP_INT_ENABLE_MASK;
4719                 WREG32(mec_int_cntl_reg, mec_int_cntl);
4720                 break;
4721         default:
4722                 break;
4723         }
4724 }
4725
4726 static int gfx_v7_0_set_priv_reg_fault_state(struct amdgpu_device *adev,
4727                                              struct amdgpu_irq_src *src,
4728                                              unsigned type,
4729                                              enum amdgpu_interrupt_state state)
4730 {
4731         u32 cp_int_cntl;
4732
4733         switch (state) {
4734         case AMDGPU_IRQ_STATE_DISABLE:
4735                 cp_int_cntl = RREG32(mmCP_INT_CNTL_RING0);
4736                 cp_int_cntl &= ~CP_INT_CNTL_RING0__PRIV_REG_INT_ENABLE_MASK;
4737                 WREG32(mmCP_INT_CNTL_RING0, cp_int_cntl);
4738                 break;
4739         case AMDGPU_IRQ_STATE_ENABLE:
4740                 cp_int_cntl = RREG32(mmCP_INT_CNTL_RING0);
4741                 cp_int_cntl |= CP_INT_CNTL_RING0__PRIV_REG_INT_ENABLE_MASK;
4742                 WREG32(mmCP_INT_CNTL_RING0, cp_int_cntl);
4743                 break;
4744         default:
4745                 break;
4746         }
4747
4748         return 0;
4749 }
4750
4751 static int gfx_v7_0_set_priv_inst_fault_state(struct amdgpu_device *adev,
4752                                               struct amdgpu_irq_src *src,
4753                                               unsigned type,
4754                                               enum amdgpu_interrupt_state state)
4755 {
4756         u32 cp_int_cntl;
4757
4758         switch (state) {
4759         case AMDGPU_IRQ_STATE_DISABLE:
4760                 cp_int_cntl = RREG32(mmCP_INT_CNTL_RING0);
4761                 cp_int_cntl &= ~CP_INT_CNTL_RING0__PRIV_INSTR_INT_ENABLE_MASK;
4762                 WREG32(mmCP_INT_CNTL_RING0, cp_int_cntl);
4763                 break;
4764         case AMDGPU_IRQ_STATE_ENABLE:
4765                 cp_int_cntl = RREG32(mmCP_INT_CNTL_RING0);
4766                 cp_int_cntl |= CP_INT_CNTL_RING0__PRIV_INSTR_INT_ENABLE_MASK;
4767                 WREG32(mmCP_INT_CNTL_RING0, cp_int_cntl);
4768                 break;
4769         default:
4770                 break;
4771         }
4772
4773         return 0;
4774 }
4775
4776 static int gfx_v7_0_set_eop_interrupt_state(struct amdgpu_device *adev,
4777                                             struct amdgpu_irq_src *src,
4778                                             unsigned type,
4779                                             enum amdgpu_interrupt_state state)
4780 {
4781         switch (type) {
4782         case AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP:
4783                 gfx_v7_0_set_gfx_eop_interrupt_state(adev, state);
4784                 break;
4785         case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP:
4786                 gfx_v7_0_set_compute_eop_interrupt_state(adev, 1, 0, state);
4787                 break;
4788         case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE1_EOP:
4789                 gfx_v7_0_set_compute_eop_interrupt_state(adev, 1, 1, state);
4790                 break;
4791         case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE2_EOP:
4792                 gfx_v7_0_set_compute_eop_interrupt_state(adev, 1, 2, state);
4793                 break;
4794         case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE3_EOP:
4795                 gfx_v7_0_set_compute_eop_interrupt_state(adev, 1, 3, state);
4796                 break;
4797         case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE0_EOP:
4798                 gfx_v7_0_set_compute_eop_interrupt_state(adev, 2, 0, state);
4799                 break;
4800         case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE1_EOP:
4801                 gfx_v7_0_set_compute_eop_interrupt_state(adev, 2, 1, state);
4802                 break;
4803         case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE2_EOP:
4804                 gfx_v7_0_set_compute_eop_interrupt_state(adev, 2, 2, state);
4805                 break;
4806         case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE3_EOP:
4807                 gfx_v7_0_set_compute_eop_interrupt_state(adev, 2, 3, state);
4808                 break;
4809         default:
4810                 break;
4811         }
4812         return 0;
4813 }
4814
4815 static int gfx_v7_0_eop_irq(struct amdgpu_device *adev,
4816                             struct amdgpu_irq_src *source,
4817                             struct amdgpu_iv_entry *entry)
4818 {
4819         u8 me_id, pipe_id;
4820         struct amdgpu_ring *ring;
4821         int i;
4822
4823         DRM_DEBUG("IH: CP EOP\n");
4824         me_id = (entry->ring_id & 0x0c) >> 2;
4825         pipe_id = (entry->ring_id & 0x03) >> 0;
4826         switch (me_id) {
4827         case 0:
4828                 amdgpu_fence_process(&adev->gfx.gfx_ring[0]);
4829                 break;
4830         case 1:
4831         case 2:
4832                 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
4833                         ring = &adev->gfx.compute_ring[i];
4834                         if ((ring->me == me_id) && (ring->pipe == pipe_id))
4835                                 amdgpu_fence_process(ring);
4836                 }
4837                 break;
4838         }
4839         return 0;
4840 }
4841
4842 static void gfx_v7_0_fault(struct amdgpu_device *adev,
4843                            struct amdgpu_iv_entry *entry)
4844 {
4845         struct amdgpu_ring *ring;
4846         u8 me_id, pipe_id;
4847         int i;
4848
4849         me_id = (entry->ring_id & 0x0c) >> 2;
4850         pipe_id = (entry->ring_id & 0x03) >> 0;
4851         switch (me_id) {
4852         case 0:
4853                 drm_sched_fault(&adev->gfx.gfx_ring[0].sched);
4854                 break;
4855         case 1:
4856         case 2:
4857                 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
4858                         ring = &adev->gfx.compute_ring[i];
4859                         if ((ring->me == me_id) && (ring->pipe == pipe_id))
4860                                 drm_sched_fault(&ring->sched);
4861                 }
4862                 break;
4863         }
4864 }
4865
4866 static int gfx_v7_0_priv_reg_irq(struct amdgpu_device *adev,
4867                                  struct amdgpu_irq_src *source,
4868                                  struct amdgpu_iv_entry *entry)
4869 {
4870         DRM_ERROR("Illegal register access in command stream\n");
4871         gfx_v7_0_fault(adev, entry);
4872         return 0;
4873 }
4874
4875 static int gfx_v7_0_priv_inst_irq(struct amdgpu_device *adev,
4876                                   struct amdgpu_irq_src *source,
4877                                   struct amdgpu_iv_entry *entry)
4878 {
4879         DRM_ERROR("Illegal instruction in command stream\n");
4880         // XXX soft reset the gfx block only
4881         gfx_v7_0_fault(adev, entry);
4882         return 0;
4883 }
4884
4885 static int gfx_v7_0_set_clockgating_state(void *handle,
4886                                           enum amd_clockgating_state state)
4887 {
4888         bool gate = false;
4889         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4890
4891         if (state == AMD_CG_STATE_GATE)
4892                 gate = true;
4893
4894         gfx_v7_0_enable_gui_idle_interrupt(adev, false);
4895         /* order matters! */
4896         if (gate) {
4897                 gfx_v7_0_enable_mgcg(adev, true);
4898                 gfx_v7_0_enable_cgcg(adev, true);
4899         } else {
4900                 gfx_v7_0_enable_cgcg(adev, false);
4901                 gfx_v7_0_enable_mgcg(adev, false);
4902         }
4903         gfx_v7_0_enable_gui_idle_interrupt(adev, true);
4904
4905         return 0;
4906 }
4907
4908 static int gfx_v7_0_set_powergating_state(void *handle,
4909                                           enum amd_powergating_state state)
4910 {
4911         bool gate = false;
4912         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4913
4914         if (state == AMD_PG_STATE_GATE)
4915                 gate = true;
4916
4917         if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG |
4918                               AMD_PG_SUPPORT_GFX_SMG |
4919                               AMD_PG_SUPPORT_GFX_DMG |
4920                               AMD_PG_SUPPORT_CP |
4921                               AMD_PG_SUPPORT_GDS |
4922                               AMD_PG_SUPPORT_RLC_SMU_HS)) {
4923                 gfx_v7_0_update_gfx_pg(adev, gate);
4924                 if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PG) {
4925                         gfx_v7_0_enable_cp_pg(adev, gate);
4926                         gfx_v7_0_enable_gds_pg(adev, gate);
4927                 }
4928         }
4929
4930         return 0;
4931 }
4932
4933 static const struct amd_ip_funcs gfx_v7_0_ip_funcs = {
4934         .name = "gfx_v7_0",
4935         .early_init = gfx_v7_0_early_init,
4936         .late_init = gfx_v7_0_late_init,
4937         .sw_init = gfx_v7_0_sw_init,
4938         .sw_fini = gfx_v7_0_sw_fini,
4939         .hw_init = gfx_v7_0_hw_init,
4940         .hw_fini = gfx_v7_0_hw_fini,
4941         .suspend = gfx_v7_0_suspend,
4942         .resume = gfx_v7_0_resume,
4943         .is_idle = gfx_v7_0_is_idle,
4944         .wait_for_idle = gfx_v7_0_wait_for_idle,
4945         .soft_reset = gfx_v7_0_soft_reset,
4946         .set_clockgating_state = gfx_v7_0_set_clockgating_state,
4947         .set_powergating_state = gfx_v7_0_set_powergating_state,
4948 };
4949
4950 static const struct amdgpu_ring_funcs gfx_v7_0_ring_funcs_gfx = {
4951         .type = AMDGPU_RING_TYPE_GFX,
4952         .align_mask = 0xff,
4953         .nop = PACKET3(PACKET3_NOP, 0x3FFF),
4954         .support_64bit_ptrs = false,
4955         .get_rptr = gfx_v7_0_ring_get_rptr,
4956         .get_wptr = gfx_v7_0_ring_get_wptr_gfx,
4957         .set_wptr = gfx_v7_0_ring_set_wptr_gfx,
4958         .emit_frame_size =
4959                 20 + /* gfx_v7_0_ring_emit_gds_switch */
4960                 7 + /* gfx_v7_0_ring_emit_hdp_flush */
4961                 5 + /* hdp invalidate */
4962                 12 + 12 + 12 + /* gfx_v7_0_ring_emit_fence_gfx x3 for user fence, vm fence */
4963                 7 + 4 + /* gfx_v7_0_ring_emit_pipeline_sync */
4964                 CIK_FLUSH_GPU_TLB_NUM_WREG * 5 + 7 + 6 + /* gfx_v7_0_ring_emit_vm_flush */
4965                 3 + 4, /* gfx_v7_ring_emit_cntxcntl including vgt flush*/
4966         .emit_ib_size = 4, /* gfx_v7_0_ring_emit_ib_gfx */
4967         .emit_ib = gfx_v7_0_ring_emit_ib_gfx,
4968         .emit_fence = gfx_v7_0_ring_emit_fence_gfx,
4969         .emit_pipeline_sync = gfx_v7_0_ring_emit_pipeline_sync,
4970         .emit_vm_flush = gfx_v7_0_ring_emit_vm_flush,
4971         .emit_gds_switch = gfx_v7_0_ring_emit_gds_switch,
4972         .emit_hdp_flush = gfx_v7_0_ring_emit_hdp_flush,
4973         .test_ring = gfx_v7_0_ring_test_ring,
4974         .test_ib = gfx_v7_0_ring_test_ib,
4975         .insert_nop = amdgpu_ring_insert_nop,
4976         .pad_ib = amdgpu_ring_generic_pad_ib,
4977         .emit_cntxcntl = gfx_v7_ring_emit_cntxcntl,
4978         .emit_wreg = gfx_v7_0_ring_emit_wreg,
4979         .soft_recovery = gfx_v7_0_ring_soft_recovery,
4980 };
4981
4982 static const struct amdgpu_ring_funcs gfx_v7_0_ring_funcs_compute = {
4983         .type = AMDGPU_RING_TYPE_COMPUTE,
4984         .align_mask = 0xff,
4985         .nop = PACKET3(PACKET3_NOP, 0x3FFF),
4986         .support_64bit_ptrs = false,
4987         .get_rptr = gfx_v7_0_ring_get_rptr,
4988         .get_wptr = gfx_v7_0_ring_get_wptr_compute,
4989         .set_wptr = gfx_v7_0_ring_set_wptr_compute,
4990         .emit_frame_size =
4991                 20 + /* gfx_v7_0_ring_emit_gds_switch */
4992                 7 + /* gfx_v7_0_ring_emit_hdp_flush */
4993                 5 + /* hdp invalidate */
4994                 7 + /* gfx_v7_0_ring_emit_pipeline_sync */
4995                 CIK_FLUSH_GPU_TLB_NUM_WREG * 5 + 7 + /* gfx_v7_0_ring_emit_vm_flush */
4996                 7 + 7 + 7, /* gfx_v7_0_ring_emit_fence_compute x3 for user fence, vm fence */
4997         .emit_ib_size = 7, /* gfx_v7_0_ring_emit_ib_compute */
4998         .emit_ib = gfx_v7_0_ring_emit_ib_compute,
4999         .emit_fence = gfx_v7_0_ring_emit_fence_compute,
5000         .emit_pipeline_sync = gfx_v7_0_ring_emit_pipeline_sync,
5001         .emit_vm_flush = gfx_v7_0_ring_emit_vm_flush,
5002         .emit_gds_switch = gfx_v7_0_ring_emit_gds_switch,
5003         .emit_hdp_flush = gfx_v7_0_ring_emit_hdp_flush,
5004         .test_ring = gfx_v7_0_ring_test_ring,
5005         .test_ib = gfx_v7_0_ring_test_ib,
5006         .insert_nop = amdgpu_ring_insert_nop,
5007         .pad_ib = amdgpu_ring_generic_pad_ib,
5008         .emit_wreg = gfx_v7_0_ring_emit_wreg,
5009 };
5010
5011 static void gfx_v7_0_set_ring_funcs(struct amdgpu_device *adev)
5012 {
5013         int i;
5014
5015         for (i = 0; i < adev->gfx.num_gfx_rings; i++)
5016                 adev->gfx.gfx_ring[i].funcs = &gfx_v7_0_ring_funcs_gfx;
5017         for (i = 0; i < adev->gfx.num_compute_rings; i++)
5018                 adev->gfx.compute_ring[i].funcs = &gfx_v7_0_ring_funcs_compute;
5019 }
5020
5021 static const struct amdgpu_irq_src_funcs gfx_v7_0_eop_irq_funcs = {
5022         .set = gfx_v7_0_set_eop_interrupt_state,
5023         .process = gfx_v7_0_eop_irq,
5024 };
5025
5026 static const struct amdgpu_irq_src_funcs gfx_v7_0_priv_reg_irq_funcs = {
5027         .set = gfx_v7_0_set_priv_reg_fault_state,
5028         .process = gfx_v7_0_priv_reg_irq,
5029 };
5030
5031 static const struct amdgpu_irq_src_funcs gfx_v7_0_priv_inst_irq_funcs = {
5032         .set = gfx_v7_0_set_priv_inst_fault_state,
5033         .process = gfx_v7_0_priv_inst_irq,
5034 };
5035
5036 static void gfx_v7_0_set_irq_funcs(struct amdgpu_device *adev)
5037 {
5038         adev->gfx.eop_irq.num_types = AMDGPU_CP_IRQ_LAST;
5039         adev->gfx.eop_irq.funcs = &gfx_v7_0_eop_irq_funcs;
5040
5041         adev->gfx.priv_reg_irq.num_types = 1;
5042         adev->gfx.priv_reg_irq.funcs = &gfx_v7_0_priv_reg_irq_funcs;
5043
5044         adev->gfx.priv_inst_irq.num_types = 1;
5045         adev->gfx.priv_inst_irq.funcs = &gfx_v7_0_priv_inst_irq_funcs;
5046 }
5047
5048 static void gfx_v7_0_set_gds_init(struct amdgpu_device *adev)
5049 {
5050         /* init asci gds info */
5051         adev->gds.gds_size = RREG32(mmGDS_VMID0_SIZE);
5052         adev->gds.gws_size = 64;
5053         adev->gds.oa_size = 16;
5054         adev->gds.gds_compute_max_wave_id = RREG32(mmGDS_COMPUTE_MAX_WAVE_ID);
5055 }
5056
5057
5058 static void gfx_v7_0_get_cu_info(struct amdgpu_device *adev)
5059 {
5060         int i, j, k, counter, active_cu_number = 0;
5061         u32 mask, bitmap, ao_bitmap, ao_cu_mask = 0;
5062         struct amdgpu_cu_info *cu_info = &adev->gfx.cu_info;
5063         unsigned disable_masks[4 * 2];
5064         u32 ao_cu_num;
5065
5066         if (adev->flags & AMD_IS_APU)
5067                 ao_cu_num = 2;
5068         else
5069                 ao_cu_num = adev->gfx.config.max_cu_per_sh;
5070
5071         memset(cu_info, 0, sizeof(*cu_info));
5072
5073         amdgpu_gfx_parse_disable_cu(disable_masks, 4, 2);
5074
5075         mutex_lock(&adev->grbm_idx_mutex);
5076         for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
5077                 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
5078                         mask = 1;
5079                         ao_bitmap = 0;
5080                         counter = 0;
5081                         gfx_v7_0_select_se_sh(adev, i, j, 0xffffffff);
5082                         if (i < 4 && j < 2)
5083                                 gfx_v7_0_set_user_cu_inactive_bitmap(
5084                                         adev, disable_masks[i * 2 + j]);
5085                         bitmap = gfx_v7_0_get_cu_active_bitmap(adev);
5086                         cu_info->bitmap[i][j] = bitmap;
5087
5088                         for (k = 0; k < adev->gfx.config.max_cu_per_sh; k ++) {
5089                                 if (bitmap & mask) {
5090                                         if (counter < ao_cu_num)
5091                                                 ao_bitmap |= mask;
5092                                         counter ++;
5093                                 }
5094                                 mask <<= 1;
5095                         }
5096                         active_cu_number += counter;
5097                         if (i < 2 && j < 2)
5098                                 ao_cu_mask |= (ao_bitmap << (i * 16 + j * 8));
5099                         cu_info->ao_cu_bitmap[i][j] = ao_bitmap;
5100                 }
5101         }
5102         gfx_v7_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
5103         mutex_unlock(&adev->grbm_idx_mutex);
5104
5105         cu_info->number = active_cu_number;
5106         cu_info->ao_cu_mask = ao_cu_mask;
5107         cu_info->simd_per_cu = NUM_SIMD_PER_CU;
5108         cu_info->max_waves_per_simd = 10;
5109         cu_info->max_scratch_slots_per_cu = 32;
5110         cu_info->wave_front_size = 64;
5111         cu_info->lds_size = 64;
5112 }
5113
5114 const struct amdgpu_ip_block_version gfx_v7_0_ip_block =
5115 {
5116         .type = AMD_IP_BLOCK_TYPE_GFX,
5117         .major = 7,
5118         .minor = 0,
5119         .rev = 0,
5120         .funcs = &gfx_v7_0_ip_funcs,
5121 };
5122
5123 const struct amdgpu_ip_block_version gfx_v7_1_ip_block =
5124 {
5125         .type = AMD_IP_BLOCK_TYPE_GFX,
5126         .major = 7,
5127         .minor = 1,
5128         .rev = 0,
5129         .funcs = &gfx_v7_0_ip_funcs,
5130 };
5131
5132 const struct amdgpu_ip_block_version gfx_v7_2_ip_block =
5133 {
5134         .type = AMD_IP_BLOCK_TYPE_GFX,
5135         .major = 7,
5136         .minor = 2,
5137         .rev = 0,
5138         .funcs = &gfx_v7_0_ip_funcs,
5139 };
5140
5141 const struct amdgpu_ip_block_version gfx_v7_3_ip_block =
5142 {
5143         .type = AMD_IP_BLOCK_TYPE_GFX,
5144         .major = 7,
5145         .minor = 3,
5146         .rev = 0,
5147         .funcs = &gfx_v7_0_ip_funcs,
5148 };