1 // SPDX-License-Identifier: GPL-2.0+
3 * PCIe host controller driver for Tegra194 SoC
5 * Copyright (C) 2019 NVIDIA Corporation.
7 * Author: Vidya Sagar <vidyas@nvidia.com>
10 #include <linux/clk.h>
11 #include <linux/debugfs.h>
12 #include <linux/delay.h>
13 #include <linux/gpio.h>
14 #include <linux/interrupt.h>
15 #include <linux/iopoll.h>
16 #include <linux/kernel.h>
17 #include <linux/module.h>
19 #include <linux/of_device.h>
20 #include <linux/of_gpio.h>
21 #include <linux/of_irq.h>
22 #include <linux/of_pci.h>
23 #include <linux/pci.h>
24 #include <linux/phy/phy.h>
25 #include <linux/pinctrl/consumer.h>
26 #include <linux/platform_device.h>
27 #include <linux/pm_runtime.h>
28 #include <linux/random.h>
29 #include <linux/reset.h>
30 #include <linux/resource.h>
31 #include <linux/types.h>
32 #include "pcie-designware.h"
33 #include <soc/tegra/bpmp.h>
34 #include <soc/tegra/bpmp-abi.h>
35 #include "../../pci.h"
37 #define APPL_PINMUX 0x0
38 #define APPL_PINMUX_PEX_RST BIT(0)
39 #define APPL_PINMUX_CLKREQ_OVERRIDE_EN BIT(2)
40 #define APPL_PINMUX_CLKREQ_OVERRIDE BIT(3)
41 #define APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE_EN BIT(4)
42 #define APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE BIT(5)
43 #define APPL_PINMUX_CLKREQ_OUT_OVRD_EN BIT(9)
44 #define APPL_PINMUX_CLKREQ_OUT_OVRD BIT(10)
47 #define APPL_CTRL_SYS_PRE_DET_STATE BIT(6)
48 #define APPL_CTRL_LTSSM_EN BIT(7)
49 #define APPL_CTRL_HW_HOT_RST_EN BIT(20)
50 #define APPL_CTRL_HW_HOT_RST_MODE_MASK GENMASK(1, 0)
51 #define APPL_CTRL_HW_HOT_RST_MODE_SHIFT 22
52 #define APPL_CTRL_HW_HOT_RST_MODE_IMDT_RST 0x1
54 #define APPL_INTR_EN_L0_0 0x8
55 #define APPL_INTR_EN_L0_0_LINK_STATE_INT_EN BIT(0)
56 #define APPL_INTR_EN_L0_0_MSI_RCV_INT_EN BIT(4)
57 #define APPL_INTR_EN_L0_0_INT_INT_EN BIT(8)
58 #define APPL_INTR_EN_L0_0_CDM_REG_CHK_INT_EN BIT(19)
59 #define APPL_INTR_EN_L0_0_SYS_INTR_EN BIT(30)
60 #define APPL_INTR_EN_L0_0_SYS_MSI_INTR_EN BIT(31)
62 #define APPL_INTR_STATUS_L0 0xC
63 #define APPL_INTR_STATUS_L0_LINK_STATE_INT BIT(0)
64 #define APPL_INTR_STATUS_L0_INT_INT BIT(8)
65 #define APPL_INTR_STATUS_L0_CDM_REG_CHK_INT BIT(18)
67 #define APPL_INTR_EN_L1_0_0 0x1C
68 #define APPL_INTR_EN_L1_0_0_LINK_REQ_RST_NOT_INT_EN BIT(1)
70 #define APPL_INTR_STATUS_L1_0_0 0x20
71 #define APPL_INTR_STATUS_L1_0_0_LINK_REQ_RST_NOT_CHGED BIT(1)
73 #define APPL_INTR_STATUS_L1_1 0x2C
74 #define APPL_INTR_STATUS_L1_2 0x30
75 #define APPL_INTR_STATUS_L1_3 0x34
76 #define APPL_INTR_STATUS_L1_6 0x3C
77 #define APPL_INTR_STATUS_L1_7 0x40
79 #define APPL_INTR_EN_L1_8_0 0x44
80 #define APPL_INTR_EN_L1_8_BW_MGT_INT_EN BIT(2)
81 #define APPL_INTR_EN_L1_8_AUTO_BW_INT_EN BIT(3)
82 #define APPL_INTR_EN_L1_8_INTX_EN BIT(11)
83 #define APPL_INTR_EN_L1_8_AER_INT_EN BIT(15)
85 #define APPL_INTR_STATUS_L1_8_0 0x4C
86 #define APPL_INTR_STATUS_L1_8_0_EDMA_INT_MASK GENMASK(11, 6)
87 #define APPL_INTR_STATUS_L1_8_0_BW_MGT_INT_STS BIT(2)
88 #define APPL_INTR_STATUS_L1_8_0_AUTO_BW_INT_STS BIT(3)
90 #define APPL_INTR_STATUS_L1_9 0x54
91 #define APPL_INTR_STATUS_L1_10 0x58
92 #define APPL_INTR_STATUS_L1_11 0x64
93 #define APPL_INTR_STATUS_L1_13 0x74
94 #define APPL_INTR_STATUS_L1_14 0x78
95 #define APPL_INTR_STATUS_L1_15 0x7C
96 #define APPL_INTR_STATUS_L1_17 0x88
98 #define APPL_INTR_EN_L1_18 0x90
99 #define APPL_INTR_EN_L1_18_CDM_REG_CHK_CMPLT BIT(2)
100 #define APPL_INTR_EN_L1_18_CDM_REG_CHK_CMP_ERR BIT(1)
101 #define APPL_INTR_EN_L1_18_CDM_REG_CHK_LOGIC_ERR BIT(0)
103 #define APPL_INTR_STATUS_L1_18 0x94
104 #define APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMPLT BIT(2)
105 #define APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMP_ERR BIT(1)
106 #define APPL_INTR_STATUS_L1_18_CDM_REG_CHK_LOGIC_ERR BIT(0)
108 #define APPL_MSI_CTRL_2 0xB0
110 #define APPL_LTR_MSG_1 0xC4
111 #define LTR_MSG_REQ BIT(15)
112 #define LTR_MST_NO_SNOOP_SHIFT 16
114 #define APPL_LTR_MSG_2 0xC8
115 #define APPL_LTR_MSG_2_LTR_MSG_REQ_STATE BIT(3)
117 #define APPL_LINK_STATUS 0xCC
118 #define APPL_LINK_STATUS_RDLH_LINK_UP BIT(0)
120 #define APPL_DEBUG 0xD0
121 #define APPL_DEBUG_PM_LINKST_IN_L2_LAT BIT(21)
122 #define APPL_DEBUG_PM_LINKST_IN_L0 0x11
123 #define APPL_DEBUG_LTSSM_STATE_MASK GENMASK(8, 3)
124 #define APPL_DEBUG_LTSSM_STATE_SHIFT 3
125 #define LTSSM_STATE_PRE_DETECT 5
127 #define APPL_RADM_STATUS 0xE4
128 #define APPL_PM_XMT_TURNOFF_STATE BIT(0)
130 #define APPL_DM_TYPE 0x100
131 #define APPL_DM_TYPE_MASK GENMASK(3, 0)
132 #define APPL_DM_TYPE_RP 0x4
133 #define APPL_DM_TYPE_EP 0x0
135 #define APPL_CFG_BASE_ADDR 0x104
136 #define APPL_CFG_BASE_ADDR_MASK GENMASK(31, 12)
138 #define APPL_CFG_IATU_DMA_BASE_ADDR 0x108
139 #define APPL_CFG_IATU_DMA_BASE_ADDR_MASK GENMASK(31, 18)
141 #define APPL_CFG_MISC 0x110
142 #define APPL_CFG_MISC_SLV_EP_MODE BIT(14)
143 #define APPL_CFG_MISC_ARCACHE_MASK GENMASK(13, 10)
144 #define APPL_CFG_MISC_ARCACHE_SHIFT 10
145 #define APPL_CFG_MISC_ARCACHE_VAL 3
147 #define APPL_CFG_SLCG_OVERRIDE 0x114
148 #define APPL_CFG_SLCG_OVERRIDE_SLCG_EN_MASTER BIT(0)
150 #define APPL_CAR_RESET_OVRD 0x12C
151 #define APPL_CAR_RESET_OVRD_CYA_OVERRIDE_CORE_RST_N BIT(0)
153 #define IO_BASE_IO_DECODE BIT(0)
154 #define IO_BASE_IO_DECODE_BIT8 BIT(8)
156 #define CFG_PREF_MEM_LIMIT_BASE_MEM_DECODE BIT(0)
157 #define CFG_PREF_MEM_LIMIT_BASE_MEM_LIMIT_DECODE BIT(16)
159 #define CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF 0x718
160 #define CFG_TIMER_CTRL_ACK_NAK_SHIFT (19)
162 #define EVENT_COUNTER_ALL_CLEAR 0x3
163 #define EVENT_COUNTER_ENABLE_ALL 0x7
164 #define EVENT_COUNTER_ENABLE_SHIFT 2
165 #define EVENT_COUNTER_EVENT_SEL_MASK GENMASK(7, 0)
166 #define EVENT_COUNTER_EVENT_SEL_SHIFT 16
167 #define EVENT_COUNTER_EVENT_Tx_L0S 0x2
168 #define EVENT_COUNTER_EVENT_Rx_L0S 0x3
169 #define EVENT_COUNTER_EVENT_L1 0x5
170 #define EVENT_COUNTER_EVENT_L1_1 0x7
171 #define EVENT_COUNTER_EVENT_L1_2 0x8
172 #define EVENT_COUNTER_GROUP_SEL_SHIFT 24
173 #define EVENT_COUNTER_GROUP_5 0x5
175 #define PORT_LOGIC_ACK_F_ASPM_CTRL 0x70C
176 #define ENTER_ASPM BIT(30)
177 #define L0S_ENTRANCE_LAT_SHIFT 24
178 #define L0S_ENTRANCE_LAT_MASK GENMASK(26, 24)
179 #define L1_ENTRANCE_LAT_SHIFT 27
180 #define L1_ENTRANCE_LAT_MASK GENMASK(29, 27)
181 #define N_FTS_SHIFT 8
182 #define N_FTS_MASK GENMASK(7, 0)
185 #define PORT_LOGIC_GEN2_CTRL 0x80C
186 #define PORT_LOGIC_GEN2_CTRL_DIRECT_SPEED_CHANGE BIT(17)
187 #define FTS_MASK GENMASK(7, 0)
190 #define PORT_LOGIC_MSI_CTRL_INT_0_EN 0x828
192 #define GEN3_EQ_CONTROL_OFF 0x8a8
193 #define GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_SHIFT 8
194 #define GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_MASK GENMASK(23, 8)
195 #define GEN3_EQ_CONTROL_OFF_FB_MODE_MASK GENMASK(3, 0)
197 #define GEN3_RELATED_OFF 0x890
198 #define GEN3_RELATED_OFF_GEN3_ZRXDC_NONCOMPL BIT(0)
199 #define GEN3_RELATED_OFF_GEN3_EQ_DISABLE BIT(16)
200 #define GEN3_RELATED_OFF_RATE_SHADOW_SEL_SHIFT 24
201 #define GEN3_RELATED_OFF_RATE_SHADOW_SEL_MASK GENMASK(25, 24)
203 #define PORT_LOGIC_AMBA_ERROR_RESPONSE_DEFAULT 0x8D0
204 #define AMBA_ERROR_RESPONSE_CRS_SHIFT 3
205 #define AMBA_ERROR_RESPONSE_CRS_MASK GENMASK(1, 0)
206 #define AMBA_ERROR_RESPONSE_CRS_OKAY 0
207 #define AMBA_ERROR_RESPONSE_CRS_OKAY_FFFFFFFF 1
208 #define AMBA_ERROR_RESPONSE_CRS_OKAY_FFFF0001 2
210 #define PORT_LOGIC_MSIX_DOORBELL 0x948
212 #define CAP_SPCIE_CAP_OFF 0x154
213 #define CAP_SPCIE_CAP_OFF_DSP_TX_PRESET0_MASK GENMASK(3, 0)
214 #define CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_MASK GENMASK(11, 8)
215 #define CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_SHIFT 8
217 #define PME_ACK_TIMEOUT 10000
219 #define LTSSM_TIMEOUT 50000 /* 50ms */
221 #define GEN3_GEN4_EQ_PRESET_INIT 5
223 #define GEN1_CORE_CLK_FREQ 62500000
224 #define GEN2_CORE_CLK_FREQ 125000000
225 #define GEN3_CORE_CLK_FREQ 250000000
226 #define GEN4_CORE_CLK_FREQ 500000000
228 static const unsigned int pcie_gen_freq[] = {
235 static const u32 event_cntr_ctrl_offset[] = {
244 static const u32 event_cntr_data_offset[] = {
253 struct tegra_pcie_dw {
255 struct resource *appl_res;
256 struct resource *dbi_res;
257 struct resource *atu_dma_res;
258 void __iomem *appl_base;
259 struct clk *core_clk;
260 struct reset_control *core_apb_rst;
261 struct reset_control *core_rst;
263 struct tegra_bpmp *bpmp;
265 bool supports_clkreq;
266 bool enable_cdm_check;
268 bool update_fc_fixup;
274 u32 cfg_link_cap_l1sub;
278 u32 aspm_l0s_enter_lat;
280 struct regulator *pex_ctl_supply;
281 struct regulator *slot_ctl_3v3;
282 struct regulator *slot_ctl_12v;
284 unsigned int phy_count;
287 struct dentry *debugfs;
290 static inline struct tegra_pcie_dw *to_tegra_pcie(struct dw_pcie *pci)
292 return container_of(pci, struct tegra_pcie_dw, pci);
295 static inline void appl_writel(struct tegra_pcie_dw *pcie, const u32 value,
298 writel_relaxed(value, pcie->appl_base + reg);
301 static inline u32 appl_readl(struct tegra_pcie_dw *pcie, const u32 reg)
303 return readl_relaxed(pcie->appl_base + reg);
306 struct tegra_pcie_soc {
307 enum dw_pcie_device_mode mode;
310 static void apply_bad_link_workaround(struct pcie_port *pp)
312 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
313 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
314 u32 current_link_width;
318 * NOTE:- Since this scenario is uncommon and link as such is not
319 * stable anyway, not waiting to confirm if link is really
320 * transitioning to Gen-2 speed
322 val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA);
323 if (val & PCI_EXP_LNKSTA_LBMS) {
324 current_link_width = (val & PCI_EXP_LNKSTA_NLW) >>
325 PCI_EXP_LNKSTA_NLW_SHIFT;
326 if (pcie->init_link_width > current_link_width) {
327 dev_warn(pci->dev, "PCIe link is bad, width reduced\n");
328 val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base +
330 val &= ~PCI_EXP_LNKCTL2_TLS;
331 val |= PCI_EXP_LNKCTL2_TLS_2_5GT;
332 dw_pcie_writew_dbi(pci, pcie->pcie_cap_base +
333 PCI_EXP_LNKCTL2, val);
335 val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base +
337 val |= PCI_EXP_LNKCTL_RL;
338 dw_pcie_writew_dbi(pci, pcie->pcie_cap_base +
339 PCI_EXP_LNKCTL, val);
344 static irqreturn_t tegra_pcie_rp_irq_handler(struct tegra_pcie_dw *pcie)
346 struct dw_pcie *pci = &pcie->pci;
347 struct pcie_port *pp = &pci->pp;
351 val = appl_readl(pcie, APPL_INTR_STATUS_L0);
352 if (val & APPL_INTR_STATUS_L0_LINK_STATE_INT) {
353 val = appl_readl(pcie, APPL_INTR_STATUS_L1_0_0);
354 if (val & APPL_INTR_STATUS_L1_0_0_LINK_REQ_RST_NOT_CHGED) {
355 appl_writel(pcie, val, APPL_INTR_STATUS_L1_0_0);
357 /* SBR & Surprise Link Down WAR */
358 val = appl_readl(pcie, APPL_CAR_RESET_OVRD);
359 val &= ~APPL_CAR_RESET_OVRD_CYA_OVERRIDE_CORE_RST_N;
360 appl_writel(pcie, val, APPL_CAR_RESET_OVRD);
362 val = appl_readl(pcie, APPL_CAR_RESET_OVRD);
363 val |= APPL_CAR_RESET_OVRD_CYA_OVERRIDE_CORE_RST_N;
364 appl_writel(pcie, val, APPL_CAR_RESET_OVRD);
366 val = dw_pcie_readl_dbi(pci, PORT_LOGIC_GEN2_CTRL);
367 val |= PORT_LOGIC_GEN2_CTRL_DIRECT_SPEED_CHANGE;
368 dw_pcie_writel_dbi(pci, PORT_LOGIC_GEN2_CTRL, val);
372 if (val & APPL_INTR_STATUS_L0_INT_INT) {
373 val = appl_readl(pcie, APPL_INTR_STATUS_L1_8_0);
374 if (val & APPL_INTR_STATUS_L1_8_0_AUTO_BW_INT_STS) {
376 APPL_INTR_STATUS_L1_8_0_AUTO_BW_INT_STS,
377 APPL_INTR_STATUS_L1_8_0);
378 apply_bad_link_workaround(pp);
380 if (val & APPL_INTR_STATUS_L1_8_0_BW_MGT_INT_STS) {
382 APPL_INTR_STATUS_L1_8_0_BW_MGT_INT_STS,
383 APPL_INTR_STATUS_L1_8_0);
385 val_w = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base +
387 dev_dbg(pci->dev, "Link Speed : Gen-%u\n", val_w &
392 val = appl_readl(pcie, APPL_INTR_STATUS_L0);
393 if (val & APPL_INTR_STATUS_L0_CDM_REG_CHK_INT) {
394 val = appl_readl(pcie, APPL_INTR_STATUS_L1_18);
395 tmp = dw_pcie_readl_dbi(pci, PCIE_PL_CHK_REG_CONTROL_STATUS);
396 if (val & APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMPLT) {
397 dev_info(pci->dev, "CDM check complete\n");
398 tmp |= PCIE_PL_CHK_REG_CHK_REG_COMPLETE;
400 if (val & APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMP_ERR) {
401 dev_err(pci->dev, "CDM comparison mismatch\n");
402 tmp |= PCIE_PL_CHK_REG_CHK_REG_COMPARISON_ERROR;
404 if (val & APPL_INTR_STATUS_L1_18_CDM_REG_CHK_LOGIC_ERR) {
405 dev_err(pci->dev, "CDM Logic error\n");
406 tmp |= PCIE_PL_CHK_REG_CHK_REG_LOGIC_ERROR;
408 dw_pcie_writel_dbi(pci, PCIE_PL_CHK_REG_CONTROL_STATUS, tmp);
409 tmp = dw_pcie_readl_dbi(pci, PCIE_PL_CHK_REG_ERR_ADDR);
410 dev_err(pci->dev, "CDM Error Address Offset = 0x%08X\n", tmp);
416 static irqreturn_t tegra_pcie_irq_handler(int irq, void *arg)
418 struct tegra_pcie_dw *pcie = arg;
420 return tegra_pcie_rp_irq_handler(pcie);
423 static int tegra_pcie_dw_rd_own_conf(struct pcie_port *pp, int where, int size,
426 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
429 * This is an endpoint mode specific register happen to appear even
430 * when controller is operating in root port mode and system hangs
431 * when it is accessed with link being in ASPM-L1 state.
432 * So skip accessing it altogether
434 if (where == PORT_LOGIC_MSIX_DOORBELL) {
436 return PCIBIOS_SUCCESSFUL;
439 return dw_pcie_read(pci->dbi_base + where, size, val);
442 static int tegra_pcie_dw_wr_own_conf(struct pcie_port *pp, int where, int size,
445 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
448 * This is an endpoint mode specific register happen to appear even
449 * when controller is operating in root port mode and system hangs
450 * when it is accessed with link being in ASPM-L1 state.
451 * So skip accessing it altogether
453 if (where == PORT_LOGIC_MSIX_DOORBELL)
454 return PCIBIOS_SUCCESSFUL;
456 return dw_pcie_write(pci->dbi_base + where, size, val);
459 #if defined(CONFIG_PCIEASPM)
460 static void disable_aspm_l11(struct tegra_pcie_dw *pcie)
464 val = dw_pcie_readl_dbi(&pcie->pci, pcie->cfg_link_cap_l1sub);
465 val &= ~PCI_L1SS_CAP_ASPM_L1_1;
466 dw_pcie_writel_dbi(&pcie->pci, pcie->cfg_link_cap_l1sub, val);
469 static void disable_aspm_l12(struct tegra_pcie_dw *pcie)
473 val = dw_pcie_readl_dbi(&pcie->pci, pcie->cfg_link_cap_l1sub);
474 val &= ~PCI_L1SS_CAP_ASPM_L1_2;
475 dw_pcie_writel_dbi(&pcie->pci, pcie->cfg_link_cap_l1sub, val);
478 static inline u32 event_counter_prog(struct tegra_pcie_dw *pcie, u32 event)
482 val = dw_pcie_readl_dbi(&pcie->pci, event_cntr_ctrl_offset[pcie->cid]);
483 val &= ~(EVENT_COUNTER_EVENT_SEL_MASK << EVENT_COUNTER_EVENT_SEL_SHIFT);
484 val |= EVENT_COUNTER_GROUP_5 << EVENT_COUNTER_GROUP_SEL_SHIFT;
485 val |= event << EVENT_COUNTER_EVENT_SEL_SHIFT;
486 val |= EVENT_COUNTER_ENABLE_ALL << EVENT_COUNTER_ENABLE_SHIFT;
487 dw_pcie_writel_dbi(&pcie->pci, event_cntr_ctrl_offset[pcie->cid], val);
488 val = dw_pcie_readl_dbi(&pcie->pci, event_cntr_data_offset[pcie->cid]);
493 static int aspm_state_cnt(struct seq_file *s, void *data)
495 struct tegra_pcie_dw *pcie = (struct tegra_pcie_dw *)
496 dev_get_drvdata(s->private);
499 seq_printf(s, "Tx L0s entry count : %u\n",
500 event_counter_prog(pcie, EVENT_COUNTER_EVENT_Tx_L0S));
502 seq_printf(s, "Rx L0s entry count : %u\n",
503 event_counter_prog(pcie, EVENT_COUNTER_EVENT_Rx_L0S));
505 seq_printf(s, "Link L1 entry count : %u\n",
506 event_counter_prog(pcie, EVENT_COUNTER_EVENT_L1));
508 seq_printf(s, "Link L1.1 entry count : %u\n",
509 event_counter_prog(pcie, EVENT_COUNTER_EVENT_L1_1));
511 seq_printf(s, "Link L1.2 entry count : %u\n",
512 event_counter_prog(pcie, EVENT_COUNTER_EVENT_L1_2));
514 /* Clear all counters */
515 dw_pcie_writel_dbi(&pcie->pci, event_cntr_ctrl_offset[pcie->cid],
516 EVENT_COUNTER_ALL_CLEAR);
518 /* Re-enable counting */
519 val = EVENT_COUNTER_ENABLE_ALL << EVENT_COUNTER_ENABLE_SHIFT;
520 val |= EVENT_COUNTER_GROUP_5 << EVENT_COUNTER_GROUP_SEL_SHIFT;
521 dw_pcie_writel_dbi(&pcie->pci, event_cntr_ctrl_offset[pcie->cid], val);
526 static void init_host_aspm(struct tegra_pcie_dw *pcie)
528 struct dw_pcie *pci = &pcie->pci;
531 val = dw_pcie_find_ext_capability(pci, PCI_EXT_CAP_ID_L1SS);
532 pcie->cfg_link_cap_l1sub = val + PCI_L1SS_CAP;
534 /* Enable ASPM counters */
535 val = EVENT_COUNTER_ENABLE_ALL << EVENT_COUNTER_ENABLE_SHIFT;
536 val |= EVENT_COUNTER_GROUP_5 << EVENT_COUNTER_GROUP_SEL_SHIFT;
537 dw_pcie_writel_dbi(pci, event_cntr_ctrl_offset[pcie->cid], val);
539 /* Program T_cmrt and T_pwr_on values */
540 val = dw_pcie_readl_dbi(pci, pcie->cfg_link_cap_l1sub);
541 val &= ~(PCI_L1SS_CAP_CM_RESTORE_TIME | PCI_L1SS_CAP_P_PWR_ON_VALUE);
542 val |= (pcie->aspm_cmrt << 8);
543 val |= (pcie->aspm_pwr_on_t << 19);
544 dw_pcie_writel_dbi(pci, pcie->cfg_link_cap_l1sub, val);
546 /* Program L0s and L1 entrance latencies */
547 val = dw_pcie_readl_dbi(pci, PORT_LOGIC_ACK_F_ASPM_CTRL);
548 val &= ~L0S_ENTRANCE_LAT_MASK;
549 val |= (pcie->aspm_l0s_enter_lat << L0S_ENTRANCE_LAT_SHIFT);
551 dw_pcie_writel_dbi(pci, PORT_LOGIC_ACK_F_ASPM_CTRL, val);
554 static int init_debugfs(struct tegra_pcie_dw *pcie)
558 d = debugfs_create_devm_seqfile(pcie->dev, "aspm_state_cnt",
559 pcie->debugfs, aspm_state_cnt);
560 if (IS_ERR_OR_NULL(d))
562 "Failed to create debugfs file \"aspm_state_cnt\"\n");
567 static inline void disable_aspm_l12(struct tegra_pcie_dw *pcie) { return; }
568 static inline void disable_aspm_l11(struct tegra_pcie_dw *pcie) { return; }
569 static inline void init_host_aspm(struct tegra_pcie_dw *pcie) { return; }
570 static inline int init_debugfs(struct tegra_pcie_dw *pcie) { return 0; }
573 static void tegra_pcie_enable_system_interrupts(struct pcie_port *pp)
575 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
576 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
580 val = appl_readl(pcie, APPL_INTR_EN_L0_0);
581 val |= APPL_INTR_EN_L0_0_LINK_STATE_INT_EN;
582 appl_writel(pcie, val, APPL_INTR_EN_L0_0);
584 val = appl_readl(pcie, APPL_INTR_EN_L1_0_0);
585 val |= APPL_INTR_EN_L1_0_0_LINK_REQ_RST_NOT_INT_EN;
586 appl_writel(pcie, val, APPL_INTR_EN_L1_0_0);
588 if (pcie->enable_cdm_check) {
589 val = appl_readl(pcie, APPL_INTR_EN_L0_0);
590 val |= APPL_INTR_EN_L0_0_CDM_REG_CHK_INT_EN;
591 appl_writel(pcie, val, APPL_INTR_EN_L0_0);
593 val = appl_readl(pcie, APPL_INTR_EN_L1_18);
594 val |= APPL_INTR_EN_L1_18_CDM_REG_CHK_CMP_ERR;
595 val |= APPL_INTR_EN_L1_18_CDM_REG_CHK_LOGIC_ERR;
596 appl_writel(pcie, val, APPL_INTR_EN_L1_18);
599 val_w = dw_pcie_readw_dbi(&pcie->pci, pcie->pcie_cap_base +
601 pcie->init_link_width = (val_w & PCI_EXP_LNKSTA_NLW) >>
602 PCI_EXP_LNKSTA_NLW_SHIFT;
604 val_w = dw_pcie_readw_dbi(&pcie->pci, pcie->pcie_cap_base +
606 val_w |= PCI_EXP_LNKCTL_LBMIE;
607 dw_pcie_writew_dbi(&pcie->pci, pcie->pcie_cap_base + PCI_EXP_LNKCTL,
611 static void tegra_pcie_enable_legacy_interrupts(struct pcie_port *pp)
613 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
614 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
617 /* Enable legacy interrupt generation */
618 val = appl_readl(pcie, APPL_INTR_EN_L0_0);
619 val |= APPL_INTR_EN_L0_0_SYS_INTR_EN;
620 val |= APPL_INTR_EN_L0_0_INT_INT_EN;
621 appl_writel(pcie, val, APPL_INTR_EN_L0_0);
623 val = appl_readl(pcie, APPL_INTR_EN_L1_8_0);
624 val |= APPL_INTR_EN_L1_8_INTX_EN;
625 val |= APPL_INTR_EN_L1_8_AUTO_BW_INT_EN;
626 val |= APPL_INTR_EN_L1_8_BW_MGT_INT_EN;
627 if (IS_ENABLED(CONFIG_PCIEAER))
628 val |= APPL_INTR_EN_L1_8_AER_INT_EN;
629 appl_writel(pcie, val, APPL_INTR_EN_L1_8_0);
632 static void tegra_pcie_enable_msi_interrupts(struct pcie_port *pp)
634 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
635 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
638 dw_pcie_msi_init(pp);
640 /* Enable MSI interrupt generation */
641 val = appl_readl(pcie, APPL_INTR_EN_L0_0);
642 val |= APPL_INTR_EN_L0_0_SYS_MSI_INTR_EN;
643 val |= APPL_INTR_EN_L0_0_MSI_RCV_INT_EN;
644 appl_writel(pcie, val, APPL_INTR_EN_L0_0);
647 static void tegra_pcie_enable_interrupts(struct pcie_port *pp)
649 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
650 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
652 /* Clear interrupt statuses before enabling interrupts */
653 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L0);
654 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_0_0);
655 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_1);
656 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_2);
657 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_3);
658 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_6);
659 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_7);
660 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_8_0);
661 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_9);
662 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_10);
663 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_11);
664 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_13);
665 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_14);
666 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_15);
667 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_17);
669 tegra_pcie_enable_system_interrupts(pp);
670 tegra_pcie_enable_legacy_interrupts(pp);
671 if (IS_ENABLED(CONFIG_PCI_MSI))
672 tegra_pcie_enable_msi_interrupts(pp);
675 static void config_gen3_gen4_eq_presets(struct tegra_pcie_dw *pcie)
677 struct dw_pcie *pci = &pcie->pci;
680 /* Program init preset */
681 for (i = 0; i < pcie->num_lanes; i++) {
682 dw_pcie_read(pci->dbi_base + CAP_SPCIE_CAP_OFF
684 val &= ~CAP_SPCIE_CAP_OFF_DSP_TX_PRESET0_MASK;
685 val |= GEN3_GEN4_EQ_PRESET_INIT;
686 val &= ~CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_MASK;
687 val |= (GEN3_GEN4_EQ_PRESET_INIT <<
688 CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_SHIFT);
689 dw_pcie_write(pci->dbi_base + CAP_SPCIE_CAP_OFF
692 offset = dw_pcie_find_ext_capability(pci,
693 PCI_EXT_CAP_ID_PL_16GT) +
695 dw_pcie_read(pci->dbi_base + offset + i, 1, &val);
696 val &= ~PCI_PL_16GT_LE_CTRL_DSP_TX_PRESET_MASK;
697 val |= GEN3_GEN4_EQ_PRESET_INIT;
698 val &= ~PCI_PL_16GT_LE_CTRL_USP_TX_PRESET_MASK;
699 val |= (GEN3_GEN4_EQ_PRESET_INIT <<
700 PCI_PL_16GT_LE_CTRL_USP_TX_PRESET_SHIFT);
701 dw_pcie_write(pci->dbi_base + offset + i, 1, val);
704 val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF);
705 val &= ~GEN3_RELATED_OFF_RATE_SHADOW_SEL_MASK;
706 dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val);
708 val = dw_pcie_readl_dbi(pci, GEN3_EQ_CONTROL_OFF);
709 val &= ~GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_MASK;
710 val |= (0x3ff << GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_SHIFT);
711 val &= ~GEN3_EQ_CONTROL_OFF_FB_MODE_MASK;
712 dw_pcie_writel_dbi(pci, GEN3_EQ_CONTROL_OFF, val);
714 val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF);
715 val &= ~GEN3_RELATED_OFF_RATE_SHADOW_SEL_MASK;
716 val |= (0x1 << GEN3_RELATED_OFF_RATE_SHADOW_SEL_SHIFT);
717 dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val);
719 val = dw_pcie_readl_dbi(pci, GEN3_EQ_CONTROL_OFF);
720 val &= ~GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_MASK;
721 val |= (0x360 << GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_SHIFT);
722 val &= ~GEN3_EQ_CONTROL_OFF_FB_MODE_MASK;
723 dw_pcie_writel_dbi(pci, GEN3_EQ_CONTROL_OFF, val);
725 val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF);
726 val &= ~GEN3_RELATED_OFF_RATE_SHADOW_SEL_MASK;
727 dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val);
730 static void tegra_pcie_prepare_host(struct pcie_port *pp)
732 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
733 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
736 val = dw_pcie_readl_dbi(pci, PCI_IO_BASE);
737 val &= ~(IO_BASE_IO_DECODE | IO_BASE_IO_DECODE_BIT8);
738 dw_pcie_writel_dbi(pci, PCI_IO_BASE, val);
740 val = dw_pcie_readl_dbi(pci, PCI_PREF_MEMORY_BASE);
741 val |= CFG_PREF_MEM_LIMIT_BASE_MEM_DECODE;
742 val |= CFG_PREF_MEM_LIMIT_BASE_MEM_LIMIT_DECODE;
743 dw_pcie_writel_dbi(pci, PCI_PREF_MEMORY_BASE, val);
745 dw_pcie_writel_dbi(pci, PCI_BASE_ADDRESS_0, 0);
748 val = dw_pcie_readl_dbi(pci, PORT_LOGIC_ACK_F_ASPM_CTRL);
749 val &= ~(N_FTS_MASK << N_FTS_SHIFT);
750 val |= N_FTS_VAL << N_FTS_SHIFT;
751 dw_pcie_writel_dbi(pci, PORT_LOGIC_ACK_F_ASPM_CTRL, val);
753 val = dw_pcie_readl_dbi(pci, PORT_LOGIC_GEN2_CTRL);
756 dw_pcie_writel_dbi(pci, PORT_LOGIC_GEN2_CTRL, val);
758 /* Enable as 0xFFFF0001 response for CRS */
759 val = dw_pcie_readl_dbi(pci, PORT_LOGIC_AMBA_ERROR_RESPONSE_DEFAULT);
760 val &= ~(AMBA_ERROR_RESPONSE_CRS_MASK << AMBA_ERROR_RESPONSE_CRS_SHIFT);
761 val |= (AMBA_ERROR_RESPONSE_CRS_OKAY_FFFF0001 <<
762 AMBA_ERROR_RESPONSE_CRS_SHIFT);
763 dw_pcie_writel_dbi(pci, PORT_LOGIC_AMBA_ERROR_RESPONSE_DEFAULT, val);
765 /* Configure Max Speed from DT */
766 if (pcie->max_speed && pcie->max_speed != -EINVAL) {
767 val = dw_pcie_readl_dbi(pci, pcie->pcie_cap_base +
769 val &= ~PCI_EXP_LNKCAP_SLS;
770 val |= pcie->max_speed;
771 dw_pcie_writel_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKCAP,
775 /* Configure Max lane width from DT */
776 val = dw_pcie_readl_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKCAP);
777 val &= ~PCI_EXP_LNKCAP_MLW;
778 val |= (pcie->num_lanes << PCI_EXP_LNKSTA_NLW_SHIFT);
779 dw_pcie_writel_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKCAP, val);
781 config_gen3_gen4_eq_presets(pcie);
783 init_host_aspm(pcie);
785 val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF);
786 val &= ~GEN3_RELATED_OFF_GEN3_ZRXDC_NONCOMPL;
787 dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val);
789 if (pcie->update_fc_fixup) {
790 val = dw_pcie_readl_dbi(pci, CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF);
791 val |= 0x1 << CFG_TIMER_CTRL_ACK_NAK_SHIFT;
792 dw_pcie_writel_dbi(pci, CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF, val);
795 dw_pcie_setup_rc(pp);
797 clk_set_rate(pcie->core_clk, GEN4_CORE_CLK_FREQ);
800 val = appl_readl(pcie, APPL_PINMUX);
801 val &= ~APPL_PINMUX_PEX_RST;
802 appl_writel(pcie, val, APPL_PINMUX);
804 usleep_range(100, 200);
807 val = appl_readl(pcie, APPL_CTRL);
808 val |= APPL_CTRL_LTSSM_EN;
809 appl_writel(pcie, val, APPL_CTRL);
812 val = appl_readl(pcie, APPL_PINMUX);
813 val |= APPL_PINMUX_PEX_RST;
814 appl_writel(pcie, val, APPL_PINMUX);
819 static int tegra_pcie_dw_host_init(struct pcie_port *pp)
821 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
822 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
823 u32 val, tmp, offset, speed;
825 tegra_pcie_prepare_host(pp);
827 if (dw_pcie_wait_for_link(pci)) {
829 * There are some endpoints which can't get the link up if
830 * root port has Data Link Feature (DLF) enabled.
831 * Refer Spec rev 4.0 ver 1.0 sec 3.4.2 & 7.7.4 for more info
832 * on Scaled Flow Control and DLF.
833 * So, need to confirm that is indeed the case here and attempt
834 * link up once again with DLF disabled.
836 val = appl_readl(pcie, APPL_DEBUG);
837 val &= APPL_DEBUG_LTSSM_STATE_MASK;
838 val >>= APPL_DEBUG_LTSSM_STATE_SHIFT;
839 tmp = appl_readl(pcie, APPL_LINK_STATUS);
840 tmp &= APPL_LINK_STATUS_RDLH_LINK_UP;
841 if (!(val == 0x11 && !tmp)) {
842 /* Link is down for all good reasons */
846 dev_info(pci->dev, "Link is down in DLL");
847 dev_info(pci->dev, "Trying again with DLFE disabled\n");
849 val = appl_readl(pcie, APPL_CTRL);
850 val &= ~APPL_CTRL_LTSSM_EN;
851 appl_writel(pcie, val, APPL_CTRL);
853 reset_control_assert(pcie->core_rst);
854 reset_control_deassert(pcie->core_rst);
856 offset = dw_pcie_find_ext_capability(pci, PCI_EXT_CAP_ID_DLF);
857 val = dw_pcie_readl_dbi(pci, offset + PCI_DLF_CAP);
858 val &= ~PCI_DLF_EXCHANGE_ENABLE;
859 dw_pcie_writel_dbi(pci, offset, val);
861 tegra_pcie_prepare_host(pp);
863 if (dw_pcie_wait_for_link(pci))
867 speed = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA) &
869 clk_set_rate(pcie->core_clk, pcie_gen_freq[speed - 1]);
871 tegra_pcie_enable_interrupts(pp);
876 static int tegra_pcie_dw_link_up(struct dw_pcie *pci)
878 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
879 u32 val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA);
881 return !!(val & PCI_EXP_LNKSTA_DLLLA);
884 static void tegra_pcie_set_msi_vec_num(struct pcie_port *pp)
886 pp->num_vectors = MAX_MSI_IRQS;
889 static const struct dw_pcie_ops tegra_dw_pcie_ops = {
890 .link_up = tegra_pcie_dw_link_up,
893 static struct dw_pcie_host_ops tegra_pcie_dw_host_ops = {
894 .rd_own_conf = tegra_pcie_dw_rd_own_conf,
895 .wr_own_conf = tegra_pcie_dw_wr_own_conf,
896 .host_init = tegra_pcie_dw_host_init,
897 .set_num_vectors = tegra_pcie_set_msi_vec_num,
900 static void tegra_pcie_disable_phy(struct tegra_pcie_dw *pcie)
902 unsigned int phy_count = pcie->phy_count;
904 while (phy_count--) {
905 phy_power_off(pcie->phys[phy_count]);
906 phy_exit(pcie->phys[phy_count]);
910 static int tegra_pcie_enable_phy(struct tegra_pcie_dw *pcie)
915 for (i = 0; i < pcie->phy_count; i++) {
916 ret = phy_init(pcie->phys[i]);
920 ret = phy_power_on(pcie->phys[i]);
929 phy_power_off(pcie->phys[i]);
931 phy_exit(pcie->phys[i]);
937 static int tegra_pcie_dw_parse_dt(struct tegra_pcie_dw *pcie)
939 struct device_node *np = pcie->dev->of_node;
942 ret = of_property_read_u32(np, "nvidia,aspm-cmrt-us", &pcie->aspm_cmrt);
944 dev_info(pcie->dev, "Failed to read ASPM T_cmrt: %d\n", ret);
948 ret = of_property_read_u32(np, "nvidia,aspm-pwr-on-t-us",
949 &pcie->aspm_pwr_on_t);
951 dev_info(pcie->dev, "Failed to read ASPM Power On time: %d\n",
954 ret = of_property_read_u32(np, "nvidia,aspm-l0s-entrance-latency-us",
955 &pcie->aspm_l0s_enter_lat);
958 "Failed to read ASPM L0s Entrance latency: %d\n", ret);
960 ret = of_property_read_u32(np, "num-lanes", &pcie->num_lanes);
962 dev_err(pcie->dev, "Failed to read num-lanes: %d\n", ret);
966 pcie->max_speed = of_pci_get_max_link_speed(np);
968 ret = of_property_read_u32_index(np, "nvidia,bpmp", 1, &pcie->cid);
970 dev_err(pcie->dev, "Failed to read Controller-ID: %d\n", ret);
974 ret = of_property_count_strings(np, "phy-names");
976 dev_err(pcie->dev, "Failed to find PHY entries: %d\n",
980 pcie->phy_count = ret;
982 if (of_property_read_bool(np, "nvidia,update-fc-fixup"))
983 pcie->update_fc_fixup = true;
985 pcie->supports_clkreq =
986 of_property_read_bool(pcie->dev->of_node, "supports-clkreq");
988 pcie->enable_cdm_check =
989 of_property_read_bool(np, "snps,enable-cdm-check");
994 static int tegra_pcie_bpmp_set_ctrl_state(struct tegra_pcie_dw *pcie,
997 struct mrq_uphy_response resp;
998 struct tegra_bpmp_message msg;
999 struct mrq_uphy_request req;
1001 /* Controller-5 doesn't need to have its state set by BPMP-FW */
1005 memset(&req, 0, sizeof(req));
1006 memset(&resp, 0, sizeof(resp));
1008 req.cmd = CMD_UPHY_PCIE_CONTROLLER_STATE;
1009 req.controller_state.pcie_controller = pcie->cid;
1010 req.controller_state.enable = enable;
1012 memset(&msg, 0, sizeof(msg));
1015 msg.tx.size = sizeof(req);
1016 msg.rx.data = &resp;
1017 msg.rx.size = sizeof(resp);
1019 return tegra_bpmp_transfer(pcie->bpmp, &msg);
1022 static void tegra_pcie_downstream_dev_to_D0(struct tegra_pcie_dw *pcie)
1024 struct pcie_port *pp = &pcie->pci.pp;
1025 struct pci_bus *child, *root_bus = NULL;
1026 struct pci_dev *pdev;
1029 * link doesn't go into L2 state with some of the endpoints with Tegra
1030 * if they are not in D0 state. So, need to make sure that immediate
1031 * downstream devices are in D0 state before sending PME_TurnOff to put
1032 * link into L2 state.
1033 * This is as per PCI Express Base r4.0 v1.0 September 27-2017,
1034 * 5.2 Link State Power Management (Page #428).
1037 list_for_each_entry(child, &pp->root_bus->children, node) {
1038 /* Bring downstream devices to D0 if they are not already in */
1039 if (child->parent == pp->root_bus) {
1046 dev_err(pcie->dev, "Failed to find downstream devices\n");
1050 list_for_each_entry(pdev, &root_bus->devices, bus_list) {
1051 if (PCI_SLOT(pdev->devfn) == 0) {
1052 if (pci_set_power_state(pdev, PCI_D0))
1054 "Failed to transition %s to D0 state\n",
1055 dev_name(&pdev->dev));
1060 static int tegra_pcie_get_slot_regulators(struct tegra_pcie_dw *pcie)
1062 pcie->slot_ctl_3v3 = devm_regulator_get_optional(pcie->dev, "vpcie3v3");
1063 if (IS_ERR(pcie->slot_ctl_3v3)) {
1064 if (PTR_ERR(pcie->slot_ctl_3v3) != -ENODEV)
1065 return PTR_ERR(pcie->slot_ctl_3v3);
1067 pcie->slot_ctl_3v3 = NULL;
1070 pcie->slot_ctl_12v = devm_regulator_get_optional(pcie->dev, "vpcie12v");
1071 if (IS_ERR(pcie->slot_ctl_12v)) {
1072 if (PTR_ERR(pcie->slot_ctl_12v) != -ENODEV)
1073 return PTR_ERR(pcie->slot_ctl_12v);
1075 pcie->slot_ctl_12v = NULL;
1081 static int tegra_pcie_enable_slot_regulators(struct tegra_pcie_dw *pcie)
1085 if (pcie->slot_ctl_3v3) {
1086 ret = regulator_enable(pcie->slot_ctl_3v3);
1089 "Failed to enable 3.3V slot supply: %d\n", ret);
1094 if (pcie->slot_ctl_12v) {
1095 ret = regulator_enable(pcie->slot_ctl_12v);
1098 "Failed to enable 12V slot supply: %d\n", ret);
1099 goto fail_12v_enable;
1104 * According to PCI Express Card Electromechanical Specification
1105 * Revision 1.1, Table-2.4, T_PVPERL (Power stable to PERST# inactive)
1106 * should be a minimum of 100ms.
1108 if (pcie->slot_ctl_3v3 || pcie->slot_ctl_12v)
1114 if (pcie->slot_ctl_3v3)
1115 regulator_disable(pcie->slot_ctl_3v3);
1119 static void tegra_pcie_disable_slot_regulators(struct tegra_pcie_dw *pcie)
1121 if (pcie->slot_ctl_12v)
1122 regulator_disable(pcie->slot_ctl_12v);
1123 if (pcie->slot_ctl_3v3)
1124 regulator_disable(pcie->slot_ctl_3v3);
1127 static int tegra_pcie_config_controller(struct tegra_pcie_dw *pcie,
1133 ret = tegra_pcie_bpmp_set_ctrl_state(pcie, true);
1136 "Failed to enable controller %u: %d\n", pcie->cid, ret);
1140 ret = tegra_pcie_enable_slot_regulators(pcie);
1142 goto fail_slot_reg_en;
1144 ret = regulator_enable(pcie->pex_ctl_supply);
1146 dev_err(pcie->dev, "Failed to enable regulator: %d\n", ret);
1150 ret = clk_prepare_enable(pcie->core_clk);
1152 dev_err(pcie->dev, "Failed to enable core clock: %d\n", ret);
1156 ret = reset_control_deassert(pcie->core_apb_rst);
1158 dev_err(pcie->dev, "Failed to deassert core APB reset: %d\n",
1160 goto fail_core_apb_rst;
1163 if (en_hw_hot_rst) {
1164 /* Enable HW_HOT_RST mode */
1165 val = appl_readl(pcie, APPL_CTRL);
1166 val &= ~(APPL_CTRL_HW_HOT_RST_MODE_MASK <<
1167 APPL_CTRL_HW_HOT_RST_MODE_SHIFT);
1168 val |= APPL_CTRL_HW_HOT_RST_EN;
1169 appl_writel(pcie, val, APPL_CTRL);
1172 ret = tegra_pcie_enable_phy(pcie);
1174 dev_err(pcie->dev, "Failed to enable PHY: %d\n", ret);
1178 /* Update CFG base address */
1179 appl_writel(pcie, pcie->dbi_res->start & APPL_CFG_BASE_ADDR_MASK,
1180 APPL_CFG_BASE_ADDR);
1182 /* Configure this core for RP mode operation */
1183 appl_writel(pcie, APPL_DM_TYPE_RP, APPL_DM_TYPE);
1185 appl_writel(pcie, 0x0, APPL_CFG_SLCG_OVERRIDE);
1187 val = appl_readl(pcie, APPL_CTRL);
1188 appl_writel(pcie, val | APPL_CTRL_SYS_PRE_DET_STATE, APPL_CTRL);
1190 val = appl_readl(pcie, APPL_CFG_MISC);
1191 val |= (APPL_CFG_MISC_ARCACHE_VAL << APPL_CFG_MISC_ARCACHE_SHIFT);
1192 appl_writel(pcie, val, APPL_CFG_MISC);
1194 if (!pcie->supports_clkreq) {
1195 val = appl_readl(pcie, APPL_PINMUX);
1196 val |= APPL_PINMUX_CLKREQ_OUT_OVRD_EN;
1197 val |= APPL_PINMUX_CLKREQ_OUT_OVRD;
1198 appl_writel(pcie, val, APPL_PINMUX);
1201 /* Update iATU_DMA base address */
1203 pcie->atu_dma_res->start & APPL_CFG_IATU_DMA_BASE_ADDR_MASK,
1204 APPL_CFG_IATU_DMA_BASE_ADDR);
1206 reset_control_deassert(pcie->core_rst);
1208 pcie->pcie_cap_base = dw_pcie_find_capability(&pcie->pci,
1211 /* Disable ASPM-L1SS advertisement as there is no CLKREQ routing */
1212 if (!pcie->supports_clkreq) {
1213 disable_aspm_l11(pcie);
1214 disable_aspm_l12(pcie);
1220 reset_control_assert(pcie->core_apb_rst);
1222 clk_disable_unprepare(pcie->core_clk);
1224 regulator_disable(pcie->pex_ctl_supply);
1226 tegra_pcie_disable_slot_regulators(pcie);
1228 tegra_pcie_bpmp_set_ctrl_state(pcie, false);
1233 static int __deinit_controller(struct tegra_pcie_dw *pcie)
1237 ret = reset_control_assert(pcie->core_rst);
1239 dev_err(pcie->dev, "Failed to assert \"core\" reset: %d\n",
1244 tegra_pcie_disable_phy(pcie);
1246 ret = reset_control_assert(pcie->core_apb_rst);
1248 dev_err(pcie->dev, "Failed to assert APB reset: %d\n", ret);
1252 clk_disable_unprepare(pcie->core_clk);
1254 ret = regulator_disable(pcie->pex_ctl_supply);
1256 dev_err(pcie->dev, "Failed to disable regulator: %d\n", ret);
1260 tegra_pcie_disable_slot_regulators(pcie);
1262 ret = tegra_pcie_bpmp_set_ctrl_state(pcie, false);
1264 dev_err(pcie->dev, "Failed to disable controller %d: %d\n",
1272 static int tegra_pcie_init_controller(struct tegra_pcie_dw *pcie)
1274 struct dw_pcie *pci = &pcie->pci;
1275 struct pcie_port *pp = &pci->pp;
1278 ret = tegra_pcie_config_controller(pcie, false);
1282 pp->ops = &tegra_pcie_dw_host_ops;
1284 ret = dw_pcie_host_init(pp);
1286 dev_err(pcie->dev, "Failed to add PCIe port: %d\n", ret);
1287 goto fail_host_init;
1293 return __deinit_controller(pcie);
1296 static int tegra_pcie_try_link_l2(struct tegra_pcie_dw *pcie)
1300 if (!tegra_pcie_dw_link_up(&pcie->pci))
1303 val = appl_readl(pcie, APPL_RADM_STATUS);
1304 val |= APPL_PM_XMT_TURNOFF_STATE;
1305 appl_writel(pcie, val, APPL_RADM_STATUS);
1307 return readl_poll_timeout_atomic(pcie->appl_base + APPL_DEBUG, val,
1308 val & APPL_DEBUG_PM_LINKST_IN_L2_LAT,
1309 1, PME_ACK_TIMEOUT);
1312 static void tegra_pcie_dw_pme_turnoff(struct tegra_pcie_dw *pcie)
1317 if (!tegra_pcie_dw_link_up(&pcie->pci)) {
1318 dev_dbg(pcie->dev, "PCIe link is not up...!\n");
1322 if (tegra_pcie_try_link_l2(pcie)) {
1323 dev_info(pcie->dev, "Link didn't transition to L2 state\n");
1325 * TX lane clock freq will reset to Gen1 only if link is in L2
1327 * So apply pex_rst to end point to force RP to go into detect
1330 data = appl_readl(pcie, APPL_PINMUX);
1331 data &= ~APPL_PINMUX_PEX_RST;
1332 appl_writel(pcie, data, APPL_PINMUX);
1334 err = readl_poll_timeout_atomic(pcie->appl_base + APPL_DEBUG,
1337 APPL_DEBUG_LTSSM_STATE_MASK) >>
1338 APPL_DEBUG_LTSSM_STATE_SHIFT) ==
1339 LTSSM_STATE_PRE_DETECT,
1342 dev_info(pcie->dev, "Link didn't go to detect state\n");
1344 /* Disable LTSSM after link is in detect state */
1345 data = appl_readl(pcie, APPL_CTRL);
1346 data &= ~APPL_CTRL_LTSSM_EN;
1347 appl_writel(pcie, data, APPL_CTRL);
1351 * DBI registers may not be accessible after this as PLL-E would be
1352 * down depending on how CLKREQ is pulled by end point
1354 data = appl_readl(pcie, APPL_PINMUX);
1355 data |= (APPL_PINMUX_CLKREQ_OVERRIDE_EN | APPL_PINMUX_CLKREQ_OVERRIDE);
1356 /* Cut REFCLK to slot */
1357 data |= APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE_EN;
1358 data &= ~APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE;
1359 appl_writel(pcie, data, APPL_PINMUX);
1362 static int tegra_pcie_deinit_controller(struct tegra_pcie_dw *pcie)
1364 tegra_pcie_downstream_dev_to_D0(pcie);
1365 dw_pcie_host_deinit(&pcie->pci.pp);
1366 tegra_pcie_dw_pme_turnoff(pcie);
1368 return __deinit_controller(pcie);
1371 static int tegra_pcie_config_rp(struct tegra_pcie_dw *pcie)
1373 struct pcie_port *pp = &pcie->pci.pp;
1374 struct device *dev = pcie->dev;
1378 if (IS_ENABLED(CONFIG_PCI_MSI)) {
1379 pp->msi_irq = of_irq_get_byname(dev->of_node, "msi");
1381 dev_err(dev, "Failed to get MSI interrupt\n");
1386 pm_runtime_enable(dev);
1388 ret = pm_runtime_get_sync(dev);
1390 dev_err(dev, "Failed to get runtime sync for PCIe dev: %d\n",
1392 goto fail_pm_get_sync;
1395 ret = pinctrl_pm_select_default_state(dev);
1397 dev_err(dev, "Failed to configure sideband pins: %d\n", ret);
1401 tegra_pcie_init_controller(pcie);
1403 pcie->link_state = tegra_pcie_dw_link_up(&pcie->pci);
1404 if (!pcie->link_state) {
1406 goto fail_host_init;
1409 name = devm_kasprintf(dev, GFP_KERNEL, "%pOFP", dev->of_node);
1412 goto fail_host_init;
1415 pcie->debugfs = debugfs_create_dir(name, NULL);
1417 dev_err(dev, "Failed to create debugfs\n");
1424 tegra_pcie_deinit_controller(pcie);
1426 pm_runtime_put_sync(dev);
1428 pm_runtime_disable(dev);
1432 static int tegra_pcie_dw_probe(struct platform_device *pdev)
1434 struct device *dev = &pdev->dev;
1435 struct resource *atu_dma_res;
1436 struct tegra_pcie_dw *pcie;
1437 struct resource *dbi_res;
1438 struct pcie_port *pp;
1439 struct dw_pcie *pci;
1445 pcie = devm_kzalloc(dev, sizeof(*pcie), GFP_KERNEL);
1450 pci->dev = &pdev->dev;
1451 pci->ops = &tegra_dw_pcie_ops;
1453 pcie->dev = &pdev->dev;
1455 ret = tegra_pcie_dw_parse_dt(pcie);
1457 dev_err(dev, "Failed to parse device tree: %d\n", ret);
1461 ret = tegra_pcie_get_slot_regulators(pcie);
1463 dev_err(dev, "Failed to get slot regulators: %d\n", ret);
1467 pcie->pex_ctl_supply = devm_regulator_get(dev, "vddio-pex-ctl");
1468 if (IS_ERR(pcie->pex_ctl_supply)) {
1469 ret = PTR_ERR(pcie->pex_ctl_supply);
1470 if (ret != -EPROBE_DEFER)
1471 dev_err(dev, "Failed to get regulator: %ld\n",
1472 PTR_ERR(pcie->pex_ctl_supply));
1476 pcie->core_clk = devm_clk_get(dev, "core");
1477 if (IS_ERR(pcie->core_clk)) {
1478 dev_err(dev, "Failed to get core clock: %ld\n",
1479 PTR_ERR(pcie->core_clk));
1480 return PTR_ERR(pcie->core_clk);
1483 pcie->appl_res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
1485 if (!pcie->appl_res) {
1486 dev_err(dev, "Failed to find \"appl\" region\n");
1490 pcie->appl_base = devm_ioremap_resource(dev, pcie->appl_res);
1491 if (IS_ERR(pcie->appl_base))
1492 return PTR_ERR(pcie->appl_base);
1494 pcie->core_apb_rst = devm_reset_control_get(dev, "apb");
1495 if (IS_ERR(pcie->core_apb_rst)) {
1496 dev_err(dev, "Failed to get APB reset: %ld\n",
1497 PTR_ERR(pcie->core_apb_rst));
1498 return PTR_ERR(pcie->core_apb_rst);
1501 phys = devm_kcalloc(dev, pcie->phy_count, sizeof(*phys), GFP_KERNEL);
1505 for (i = 0; i < pcie->phy_count; i++) {
1506 name = kasprintf(GFP_KERNEL, "p2u-%u", i);
1508 dev_err(dev, "Failed to create P2U string\n");
1511 phys[i] = devm_phy_get(dev, name);
1513 if (IS_ERR(phys[i])) {
1514 ret = PTR_ERR(phys[i]);
1515 if (ret != -EPROBE_DEFER)
1516 dev_err(dev, "Failed to get PHY: %d\n", ret);
1523 dbi_res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "dbi");
1525 dev_err(dev, "Failed to find \"dbi\" region\n");
1528 pcie->dbi_res = dbi_res;
1530 pci->dbi_base = devm_ioremap_resource(dev, dbi_res);
1531 if (IS_ERR(pci->dbi_base))
1532 return PTR_ERR(pci->dbi_base);
1534 /* Tegra HW locates DBI2 at a fixed offset from DBI */
1535 pci->dbi_base2 = pci->dbi_base + 0x1000;
1537 atu_dma_res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
1540 dev_err(dev, "Failed to find \"atu_dma\" region\n");
1543 pcie->atu_dma_res = atu_dma_res;
1545 pci->atu_base = devm_ioremap_resource(dev, atu_dma_res);
1546 if (IS_ERR(pci->atu_base))
1547 return PTR_ERR(pci->atu_base);
1549 pcie->core_rst = devm_reset_control_get(dev, "core");
1550 if (IS_ERR(pcie->core_rst)) {
1551 dev_err(dev, "Failed to get core reset: %ld\n",
1552 PTR_ERR(pcie->core_rst));
1553 return PTR_ERR(pcie->core_rst);
1556 pp->irq = platform_get_irq_byname(pdev, "intr");
1558 dev_err(dev, "Failed to get \"intr\" interrupt\n");
1562 ret = devm_request_irq(dev, pp->irq, tegra_pcie_irq_handler,
1563 IRQF_SHARED, "tegra-pcie-intr", pcie);
1565 dev_err(dev, "Failed to request IRQ %d: %d\n", pp->irq, ret);
1569 pcie->bpmp = tegra_bpmp_get(dev);
1570 if (IS_ERR(pcie->bpmp))
1571 return PTR_ERR(pcie->bpmp);
1573 platform_set_drvdata(pdev, pcie);
1575 ret = tegra_pcie_config_rp(pcie);
1576 if (ret && ret != -ENOMEDIUM)
1582 tegra_bpmp_put(pcie->bpmp);
1586 static int tegra_pcie_dw_remove(struct platform_device *pdev)
1588 struct tegra_pcie_dw *pcie = platform_get_drvdata(pdev);
1590 if (!pcie->link_state)
1593 debugfs_remove_recursive(pcie->debugfs);
1594 tegra_pcie_deinit_controller(pcie);
1595 pm_runtime_put_sync(pcie->dev);
1596 pm_runtime_disable(pcie->dev);
1597 tegra_bpmp_put(pcie->bpmp);
1602 static int tegra_pcie_dw_suspend_late(struct device *dev)
1604 struct tegra_pcie_dw *pcie = dev_get_drvdata(dev);
1607 if (!pcie->link_state)
1610 /* Enable HW_HOT_RST mode */
1611 val = appl_readl(pcie, APPL_CTRL);
1612 val &= ~(APPL_CTRL_HW_HOT_RST_MODE_MASK <<
1613 APPL_CTRL_HW_HOT_RST_MODE_SHIFT);
1614 val |= APPL_CTRL_HW_HOT_RST_EN;
1615 appl_writel(pcie, val, APPL_CTRL);
1620 static int tegra_pcie_dw_suspend_noirq(struct device *dev)
1622 struct tegra_pcie_dw *pcie = dev_get_drvdata(dev);
1624 if (!pcie->link_state)
1627 /* Save MSI interrupt vector */
1628 pcie->msi_ctrl_int = dw_pcie_readl_dbi(&pcie->pci,
1629 PORT_LOGIC_MSI_CTRL_INT_0_EN);
1630 tegra_pcie_downstream_dev_to_D0(pcie);
1631 tegra_pcie_dw_pme_turnoff(pcie);
1633 return __deinit_controller(pcie);
1636 static int tegra_pcie_dw_resume_noirq(struct device *dev)
1638 struct tegra_pcie_dw *pcie = dev_get_drvdata(dev);
1641 if (!pcie->link_state)
1644 ret = tegra_pcie_config_controller(pcie, true);
1648 ret = tegra_pcie_dw_host_init(&pcie->pci.pp);
1650 dev_err(dev, "Failed to init host: %d\n", ret);
1651 goto fail_host_init;
1654 /* Restore MSI interrupt vector */
1655 dw_pcie_writel_dbi(&pcie->pci, PORT_LOGIC_MSI_CTRL_INT_0_EN,
1656 pcie->msi_ctrl_int);
1661 return __deinit_controller(pcie);
1664 static int tegra_pcie_dw_resume_early(struct device *dev)
1666 struct tegra_pcie_dw *pcie = dev_get_drvdata(dev);
1669 if (!pcie->link_state)
1672 /* Disable HW_HOT_RST mode */
1673 val = appl_readl(pcie, APPL_CTRL);
1674 val &= ~(APPL_CTRL_HW_HOT_RST_MODE_MASK <<
1675 APPL_CTRL_HW_HOT_RST_MODE_SHIFT);
1676 val |= APPL_CTRL_HW_HOT_RST_MODE_IMDT_RST <<
1677 APPL_CTRL_HW_HOT_RST_MODE_SHIFT;
1678 val &= ~APPL_CTRL_HW_HOT_RST_EN;
1679 appl_writel(pcie, val, APPL_CTRL);
1684 static void tegra_pcie_dw_shutdown(struct platform_device *pdev)
1686 struct tegra_pcie_dw *pcie = platform_get_drvdata(pdev);
1688 if (!pcie->link_state)
1691 debugfs_remove_recursive(pcie->debugfs);
1692 tegra_pcie_downstream_dev_to_D0(pcie);
1694 disable_irq(pcie->pci.pp.irq);
1695 if (IS_ENABLED(CONFIG_PCI_MSI))
1696 disable_irq(pcie->pci.pp.msi_irq);
1698 tegra_pcie_dw_pme_turnoff(pcie);
1699 __deinit_controller(pcie);
1702 static const struct of_device_id tegra_pcie_dw_of_match[] = {
1704 .compatible = "nvidia,tegra194-pcie",
1709 static const struct dev_pm_ops tegra_pcie_dw_pm_ops = {
1710 .suspend_late = tegra_pcie_dw_suspend_late,
1711 .suspend_noirq = tegra_pcie_dw_suspend_noirq,
1712 .resume_noirq = tegra_pcie_dw_resume_noirq,
1713 .resume_early = tegra_pcie_dw_resume_early,
1716 static struct platform_driver tegra_pcie_dw_driver = {
1717 .probe = tegra_pcie_dw_probe,
1718 .remove = tegra_pcie_dw_remove,
1719 .shutdown = tegra_pcie_dw_shutdown,
1721 .name = "tegra194-pcie",
1722 .pm = &tegra_pcie_dw_pm_ops,
1723 .of_match_table = tegra_pcie_dw_of_match,
1726 module_platform_driver(tegra_pcie_dw_driver);
1728 MODULE_DEVICE_TABLE(of, tegra_pcie_dw_of_match);
1730 MODULE_AUTHOR("Vidya Sagar <vidyas@nvidia.com>");
1731 MODULE_DESCRIPTION("NVIDIA PCIe host controller driver");
1732 MODULE_LICENSE("GPL v2");