Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 548e5db3 authored by Tony Truong's avatar Tony Truong
Browse files

msm: pcie: add PCIe bridge support 



PCIe Endpoints can be connected to PCIe Root Complex via PCIe
bridges. Add the support in PCIe bus driver to configure and
manage PCIe bridge.

Change-Id: I31c3d8bed3be5550be8b6b304533d35bbbf42932
Signed-off-by: default avatarTony Truong <truong@codeaurora.org>
parent b2493d05

drivers/pci/host/pci-msm.c

+382 −122
Original line number Diff line number Diff line
@@ -173,6 +173,7 @@ 
#define PCIE20_CAP_LINKCTRLSTATUS	(PCIE20_CAP + 0x10)



#define PCIE20_COMMAND_STATUS	    0x04

#define PCIE20_HEADER_TYPE		0x0C

#define PCIE20_BUSNUMBERS		  0x18

#define PCIE20_MEMORY_BASE_LIMIT	 0x20

#define PCIE20_L1SUB_CONTROL1	    0x158
@@ -191,6 +192,8 @@ 
#define PCIE20_PLR_IATU_LTAR	     0x918

#define PCIE20_PLR_IATU_UTAR	     0x91c



#define PCIE20_CTRL1_TYPE_CFG0		0x04

#define PCIE20_CTRL1_TYPE_CFG1		0x05



#define PCIE_VENDOR_ID_RCP		 0x17cb

#define PCIE_DEVICE_ID_RCP		 0x0300
@@ -224,6 +227,7 @@ 
#define MSM_PCIE_MAX_CLK 7

#define MSM_PCIE_MAX_PIPE_CLK 1

#define MAX_RC_NUM 2

#define MAX_DEVICE_NUM 20

#define PCIE_MSI_NR_IRQS 256

#define PCIE_LOG_PAGES (50)

#define PCIE_CONF_SPACE_DW			1024
@@ -254,8 +258,8 @@ 
#define PCIE_LOWER_ADDR(addr) ((u32)((addr) & 0xffffffff))



/* Config Space Offsets */

#define BDF_OFFSET(bus, device, function) \

	((bus << 24) | (device << 16) | (function << 8))

#define BDF_OFFSET(bus, devfn) \

	((bus << 24) | (devfn << 16))



#define PCIE_BUS_PRIV_DATA(pdev) \

	(((struct pci_sys_data *)pdev->bus->sysdata)->private_data)
@@ -364,6 +368,15 @@ struct msm_pcie_irq_info_t { 
	uint32_t	    num;

};



/* PCIe device info structure */

struct msm_pcie_device_info {

	u32			bdf;

	struct pci_dev		*dev;

	int			domain;

	void __iomem		*conf_base;

	unsigned long		phy_address;

};



/* msm pcie device structure */

struct msm_pcie_dev_t {

	struct platform_device	 *pdev;
@@ -422,6 +435,7 @@ struct msm_pcie_dev_t { 
	uint32_t			   n_fts;

	bool				 ext_ref_clk;

	uint32_t			   ep_latency;

	uint32_t			current_bdf;

	bool				 ep_wakeirq;



	uint32_t			   rc_idx;
@@ -434,12 +448,13 @@ struct msm_pcie_dev_t { 
	ulong				linkdown_counter;

	bool				 suspending;

	ulong				wake_counter;

	u32			     ep_shadow[PCIE_CONF_SPACE_DW];

	u32		ep_shadow[MAX_DEVICE_NUM][PCIE_CONF_SPACE_DW];

	u32				  rc_shadow[PCIE_CONF_SPACE_DW];

	bool				 shadow_en;

	struct msm_pcie_register_event *event_reg;

	bool				 power_on;

	void				 *ipc_log;

	struct msm_pcie_device_info   pcidev_table[MAX_DEVICE_NUM];

};
@@ -448,6 +463,9 @@ static int msm_pcie_debug_mask; 
module_param_named(debug_mask, msm_pcie_debug_mask,

			    int, S_IRUGO | S_IWUSR | S_IWGRP);



/* Table to track info of PCIe devices */

static struct msm_pcie_device_info

	msm_pcie_dev_tbl[MAX_RC_NUM * MAX_DEVICE_NUM];



/* PCIe driver state */

struct pcie_drv_sta {
@@ -788,32 +806,45 @@ static bool msm_pcie_confirm_linkup(struct msm_pcie_dev_t *dev, 


static void msm_pcie_cfg_recover(struct msm_pcie_dev_t *dev, bool rc)

{

	int i;

	int i, j;

	u32 val = 0;

	u32 *shadow;

	void *cfg;

	void *cfg = dev->conf;



	for (i = 0; i < MAX_DEVICE_NUM; i++) {

		if (!rc && !dev->pcidev_table[i].bdf)

			break;

		if (rc) {

		shadow = dev->rc_shadow;

			cfg = dev->dm_core;

			shadow = dev->rc_shadow;

		} else {

		shadow = dev->ep_shadow;

		cfg = dev->conf;

			shadow = dev->ep_shadow[i];

			PCIE_DBG(dev,

				"PCIe Device: %02x:%02x.%01x\n",

				dev->pcidev_table[i].bdf >> 24,

				dev->pcidev_table[i].bdf >> 19 & 0x1f,

				dev->pcidev_table[i].bdf >> 16 & 0x07);

		}



	for (i = PCIE_CONF_SPACE_DW - 1; i >= 0; i--) {

		val = readl_relaxed(shadow + i);

		for (j = PCIE_CONF_SPACE_DW - 1; j >= 0; j--) {

			val = readl_relaxed(shadow + j);

			if (val != PCIE_CLEAR) {

			PCIE_DBG(dev, "PCIe: before recovery:cfg 0x%x:0x%x\n",

				i * 4, readl_relaxed(cfg + i * 4));

			PCIE_DBG(dev, "PCIe: shadow_dw[%d]:cfg 0x%x:0x%x\n",

				i, i * 4, val);

			writel_relaxed(val, cfg + i * 4);

				PCIE_DBG(dev,

					"PCIe: before recovery:cfg 0x%x:0x%x\n",

					j * 4, readl_relaxed(cfg + j * 4));

				PCIE_DBG(dev,

					"PCIe: shadow_dw[%d]:cfg 0x%x:0x%x\n",

					j, j * 4, val);

				writel_relaxed(val, cfg + j * 4);

				wmb();

			PCIE_DBG(dev, "PCIe: after recovery:cfg 0x%x:0x%x\n\n",

				i * 4, readl_relaxed(cfg + i * 4));

				PCIE_DBG(dev,

					"PCIe: after recovery:cfg 0x%x:0x%x\n\n",

					j * 4, readl_relaxed(cfg + j * 4));

			}

		}

		if (rc)

			break;

		cfg += SZ_4K;

	}

}



static void msm_pcie_write_mask(void __iomem *addr,
@@ -832,6 +863,144 @@ static inline int msm_pcie_is_link_up(struct msm_pcie_dev_t *dev) 
			PCIE20_CAP_LINKCTRLSTATUS) & BIT(29);

}



/**

 * msm_pcie_iatu_config - configure outbound address translation region

 * @dev: root commpex

 * @nr: region number

 * @type: target transaction type, see PCIE20_CTRL1_TYPE_xxx

 * @host_addr: - region start address on host

 * @host_end: - region end address (low 32 bit) on host,

 *	upper 32 bits are same as for @host_addr

 * @target_addr: - region start address on target

 */

static void msm_pcie_iatu_config(struct msm_pcie_dev_t *dev, int nr, u8 type,

				unsigned long host_addr, u32 host_end,

				unsigned long target_addr)

{

	void __iomem *pcie20 = dev->dm_core;



	if (dev->shadow_en) {

		writel_relaxed(nr, dev->rc_shadow +

				PCIE20_PLR_IATU_VIEWPORT / 4);

		writel_relaxed(type, dev->rc_shadow +

				PCIE20_PLR_IATU_CTRL1 / 4);

		writel_relaxed(lower_32_bits(host_addr), dev->rc_shadow +

				PCIE20_PLR_IATU_LBAR / 4);

		writel_relaxed(upper_32_bits(host_addr), dev->rc_shadow +

				PCIE20_PLR_IATU_UBAR / 4);

		writel_relaxed(host_end, dev->rc_shadow +

				PCIE20_PLR_IATU_LAR / 4);

		writel_relaxed(lower_32_bits(target_addr), dev->rc_shadow +

				PCIE20_PLR_IATU_LTAR / 4);

		writel_relaxed(upper_32_bits(target_addr), dev->rc_shadow +

				PCIE20_PLR_IATU_UTAR / 4);

		writel_relaxed(BIT(31), dev->rc_shadow +

				PCIE20_PLR_IATU_CTRL2 / 4);

	}



	/* select region */

	writel_relaxed(nr, pcie20 + PCIE20_PLR_IATU_VIEWPORT);

	/* ensure that hardware locks it */

	wmb();



	/* switch off region before changing it */

	writel_relaxed(0, pcie20 + PCIE20_PLR_IATU_CTRL2);

	/* and wait till it propagates to the hardware */

	wmb();



	writel_relaxed(type, pcie20 + PCIE20_PLR_IATU_CTRL1);

	writel_relaxed(lower_32_bits(host_addr),

		       pcie20 + PCIE20_PLR_IATU_LBAR);

	writel_relaxed(upper_32_bits(host_addr),

		       pcie20 + PCIE20_PLR_IATU_UBAR);

	writel_relaxed(host_end, pcie20 + PCIE20_PLR_IATU_LAR);

	writel_relaxed(lower_32_bits(target_addr),

		       pcie20 + PCIE20_PLR_IATU_LTAR);

	writel_relaxed(upper_32_bits(target_addr),

		       pcie20 + PCIE20_PLR_IATU_UTAR);

	wmb();

	writel_relaxed(BIT(31), pcie20 + PCIE20_PLR_IATU_CTRL2);



	/* ensure that changes propagated to the hardware */

	wmb();



	if (dev->enumerated) {

		PCIE_DBG(dev, "IATU for Endpoint %02x:%02x.%01x\n",

			dev->pcidev_table[nr].bdf >> 24,

			dev->pcidev_table[nr].bdf >> 19 & 0x1f,

			dev->pcidev_table[nr].bdf >> 16 & 0x07);

		PCIE_DBG(dev, "PCIE20_PLR_IATU_VIEWPORT:0x%x\n",

			readl_relaxed(dev->dm_core + PCIE20_PLR_IATU_VIEWPORT));

		PCIE_DBG(dev, "PCIE20_PLR_IATU_CTRL1:0x%x\n",

			readl_relaxed(dev->dm_core + PCIE20_PLR_IATU_CTRL1));

		PCIE_DBG(dev, "PCIE20_PLR_IATU_LBAR:0x%x\n",

			readl_relaxed(dev->dm_core + PCIE20_PLR_IATU_LBAR));

		PCIE_DBG(dev, "PCIE20_PLR_IATU_UBAR:0x%x\n",

			readl_relaxed(dev->dm_core + PCIE20_PLR_IATU_UBAR));

		PCIE_DBG(dev, "PCIE20_PLR_IATU_LAR:0x%x\n",

			readl_relaxed(dev->dm_core + PCIE20_PLR_IATU_LAR));

		PCIE_DBG(dev, "PCIE20_PLR_IATU_LTAR:0x%x\n",

			readl_relaxed(dev->dm_core + PCIE20_PLR_IATU_LTAR));

		PCIE_DBG(dev, "PCIE20_PLR_IATU_UTAR:0x%x\n",

			readl_relaxed(dev->dm_core + PCIE20_PLR_IATU_UTAR));

		PCIE_DBG(dev, "PCIE20_PLR_IATU_CTRL2:0x%x\n\n",

			readl_relaxed(dev->dm_core + PCIE20_PLR_IATU_CTRL2));

	}

}



/**

 * msm_pcie_cfg_bdf - configure for config access

 * @dev: root commpex

 * @bus: PCI bus number

 * @devfn: PCI dev and function number

 *

 * Remap if required region 0 for config access of proper type

 * (CFG0 for bus 1, CFG1 for other buses)

 * Cache current device bdf for speed-up

 */

static void msm_pcie_cfg_bdf(struct msm_pcie_dev_t *dev, u8 bus, u8 devfn)

{

	struct resource *axi_conf = dev->res[MSM_PCIE_RES_CONF].resource;

	u32 bdf  = BDF_OFFSET(bus, devfn);

	u8 type = bus == 1 ? PCIE20_CTRL1_TYPE_CFG0 : PCIE20_CTRL1_TYPE_CFG1;

	if (dev->current_bdf == bdf)

		return;



	msm_pcie_iatu_config(dev, 0, type,

			axi_conf->start,

			axi_conf->start + SZ_4K - 1,

			bdf);



	dev->current_bdf = bdf;

}



static inline void msm_pcie_save_shadow(struct msm_pcie_dev_t *dev,

					u32 word_offset, u32 wr_val,

					u32 bdf, bool rc)

{

	int i, j;

	u32 max_dev = MAX_RC_NUM * MAX_DEVICE_NUM;



	if (rc) {

		dev->rc_shadow[word_offset / 4] = wr_val;

	} else {

		for (i = 0; i < MAX_DEVICE_NUM; i++) {

			if (!dev->pcidev_table[i].bdf) {

				for (j = 0; j < max_dev; j++)

					if (!msm_pcie_dev_tbl[j].bdf) {

						msm_pcie_dev_tbl[j].bdf = bdf;

						break;

					}

				dev->pcidev_table[i].bdf = bdf;

			}

			if (dev->pcidev_table[i].bdf == bdf) {

				dev->ep_shadow[i][word_offset / 4] = wr_val;

				break;

			}

		}

	}

}



static inline int msm_pcie_oper_conf(struct pci_bus *bus, u32 devfn, int oper,

				     int where, int size, u32 *val)

{
@@ -842,6 +1011,8 @@ static inline int msm_pcie_oper_conf(struct pci_bus *bus, u32 devfn, int oper, 
	bool rc = false;

	u32 rc_idx;

	int rv = 0;

	u32 bdf = BDF_OFFSET(bus->number, devfn);

	int i;



	dev = ((struct msm_pcie_dev_t *)

		(((struct pci_sys_data *)bus->sysdata)->private_data));
@@ -853,29 +1024,8 @@ static inline int msm_pcie_oper_conf(struct pci_bus *bus, u32 devfn, int oper, 
		goto out;

	}



	/* Do the bus->number based access control since we don't support

	   ECAM mechanism */



	switch (bus->number) {

	case 0:

		rc = true;

	case 1:

	rc_idx = dev->rc_idx;

		break;

	default:

		PCIE_ERR(dev, "PCIe: unsupported bus number:%d\n", bus->number);

		*val = ~0;

		rv = PCIBIOS_DEVICE_NOT_FOUND;

		goto out;

	}



	if ((bus->number > MAX_BUS_NUM) || (devfn != 0)) {

		PCIE_DBG(dev, "RC%d invalid %s - bus %d devfn %d\n", rc_idx,

			 (oper == RD) ? "rd" : "wr", bus->number, devfn);

		*val = ~0;

		rv = PCIBIOS_DEVICE_NOT_FOUND;

		goto out;

	}

	rc = (bus->number == 0);



	spin_lock_irqsave(&dev->cfg_lock, dev->irqsave_flags);
@@ -887,6 +1037,14 @@ static inline int msm_pcie_oper_conf(struct pci_bus *bus, u32 devfn, int oper, 
		goto unlock;

	}



	if (rc && (devfn != 0)) {

		PCIE_DBG(dev, "RC%d invalid %s - bus %d devfn %d\n", rc_idx,

			 (oper == RD) ? "rd" : "wr", bus->number, devfn);

		*val = ~0;

		rv = PCIBIOS_DEVICE_NOT_FOUND;

		goto unlock;

	}



	if (dev->link_status != MSM_PCIE_LINK_ENABLED) {

		PCIE_DBG(dev,

			"Access to RC%d %d:0x%02x + 0x%04x[%d] is denied because link is down\n",
@@ -907,11 +1065,29 @@ static inline int msm_pcie_oper_conf(struct pci_bus *bus, u32 devfn, int oper, 
			goto unlock;

	}



	if (!rc && !dev->enumerated)

		msm_pcie_cfg_bdf(dev, bus->number, devfn);



	word_offset = where & ~0x3;

	byte_offset = where & 0x3;

	mask = (~0 >> (8 * (4 - size))) << (8 * byte_offset);



	if (rc || !dev->enumerated) {

		config_base = rc ? dev->dm_core : dev->conf;

	} else {

		for (i = 0; i < MAX_DEVICE_NUM; i++) {

			if (dev->pcidev_table[i].bdf == bdf) {

				config_base = dev->pcidev_table[i].conf_base;

				break;

			}

		}

		if (i == MAX_DEVICE_NUM) {

			*val = ~0;

			rv = PCIBIOS_DEVICE_NOT_FOUND;

			goto unlock;

		}

	}



	rd_val = readl_relaxed(config_base + word_offset);



	if (oper == RD) {
@@ -925,16 +1101,12 @@ static inline int msm_pcie_oper_conf(struct pci_bus *bus, u32 devfn, int oper, 
		writel_relaxed(wr_val, config_base + word_offset);

		wmb(); /* ensure config data is written to hardware register */



		if (rd_val == PCIE_LINK_DOWN) {

		if (rd_val == PCIE_LINK_DOWN)

			PCIE_ERR(dev,

				"Read of RC%d %d:0x%02x + 0x%04x[%d] is all FFs\n",

				rc_idx, bus->number, devfn, where, size);

		} else if (dev->shadow_en) {

			if (rc)

				dev->rc_shadow[word_offset / 4] = wr_val;

			else

				dev->ep_shadow[word_offset / 4] = wr_val;

		}

		else if (dev->shadow_en)

			msm_pcie_save_shadow(dev, word_offset, wr_val, bdf, rc);



		PCIE_DBG(dev,

			"RC%d %d:0x%02x + 0x%04x[%d] <- 0x%08x; rd 0x%08x val 0x%08x\n",
@@ -1259,77 +1431,40 @@ static void msm_pcie_pipe_clk_deinit(struct msm_pcie_dev_t *dev) 
				dev->pipeclk[i].hdl);

}





static void msm_pcie_config_controller(struct msm_pcie_dev_t *dev)

static void msm_pcie_iatu_config_all_ep(struct msm_pcie_dev_t *dev)

{

	struct resource *axi_conf = dev->res[MSM_PCIE_RES_CONF].resource;

	u32 dev_conf, upper, lower, limit;



	PCIE_DBG(dev, "RC%d\n", dev->rc_idx);

	int i;

	u8 type;

	struct msm_pcie_device_info *dev_table = dev->pcidev_table;



	if (IS_ENABLED(CONFIG_ARM_LPAE)) {

		lower = PCIE_LOWER_ADDR(axi_conf->start);

		upper = PCIE_UPPER_ADDR(axi_conf->start);

		limit = PCIE_LOWER_ADDR(axi_conf->end);

	} else {

		lower = axi_conf->start;

		upper = 0;

		limit = axi_conf->end;

	}

	for (i = 0; i < MAX_DEVICE_NUM; i++) {

		if (!dev_table[i].bdf)

			break;



	dev_conf = BDF_OFFSET(1, 0, 0);

		type = dev_table[i].bdf >> 24 == 0x1 ?

			PCIE20_CTRL1_TYPE_CFG0 : PCIE20_CTRL1_TYPE_CFG1;



	if (dev->shadow_en) {

		writel_relaxed(0, dev->rc_shadow +

				PCIE20_PLR_IATU_VIEWPORT / 4);

		writel_relaxed(4, dev->rc_shadow +

				PCIE20_PLR_IATU_CTRL1 / 4);

		writel_relaxed(lower, dev->rc_shadow +

				PCIE20_PLR_IATU_LBAR / 4);

		writel_relaxed(upper, dev->rc_shadow +

				PCIE20_PLR_IATU_UBAR / 4);

		writel_relaxed(limit, dev->rc_shadow + PCIE20_PLR_IATU_LAR / 4);

		writel_relaxed(dev_conf, dev->rc_shadow +

				PCIE20_PLR_IATU_LTAR / 4);

		writel_relaxed(0, dev->rc_shadow + PCIE20_PLR_IATU_UTAR / 4);

		writel_relaxed(BIT(31), dev->rc_shadow +

				PCIE20_PLR_IATU_CTRL2 / 4);

		msm_pcie_iatu_config(dev, i, type, dev_table[i].phy_address,

			dev_table[i].phy_address + SZ_4K - 1,

			dev_table[i].bdf);

	}

}



static void msm_pcie_config_controller(struct msm_pcie_dev_t *dev)

{

	PCIE_DBG(dev, "RC%d\n", dev->rc_idx);



	/*

	 * program and enable address translation region 0 (device config

	 * address space); region type config;

	 * axi config address range to device config address range

	 */

	writel_relaxed(0, dev->dm_core + PCIE20_PLR_IATU_VIEWPORT);

	/* ensure that hardware locks the region before programming it */

	wmb();



	writel_relaxed(4, dev->dm_core + PCIE20_PLR_IATU_CTRL1);

	writel_relaxed(lower, dev->dm_core + PCIE20_PLR_IATU_LBAR);

	writel_relaxed(upper, dev->dm_core + PCIE20_PLR_IATU_UBAR);

	writel_relaxed(limit, dev->dm_core + PCIE20_PLR_IATU_LAR);

	writel_relaxed(dev_conf, dev->dm_core + PCIE20_PLR_IATU_LTAR);

	writel_relaxed(0, dev->dm_core + PCIE20_PLR_IATU_UTAR);

	writel_relaxed(BIT(31), dev->dm_core + PCIE20_PLR_IATU_CTRL2);

	/* ensure that hardware registers the configuration */

	wmb();

	PCIE_DBG(dev, "PCIE20_PLR_IATU_VIEWPORT:0x%x\n",

		readl_relaxed(dev->dm_core + PCIE20_PLR_IATU_VIEWPORT));

	PCIE_DBG(dev, "PCIE20_PLR_IATU_CTRL1:0x%x\n",

		readl_relaxed(dev->dm_core + PCIE20_PLR_IATU_CTRL1));

	PCIE_DBG(dev, "PCIE20_PLR_IATU_LBAR:0x%x\n",

		readl_relaxed(dev->dm_core + PCIE20_PLR_IATU_LBAR));

	PCIE_DBG(dev, "PCIE20_PLR_IATU_UBAR:0x%x\n",

		readl_relaxed(dev->dm_core + PCIE20_PLR_IATU_UBAR));

	PCIE_DBG(dev, "PCIE20_PLR_IATU_LAR:0x%x\n",

		readl_relaxed(dev->dm_core + PCIE20_PLR_IATU_LAR));

	PCIE_DBG(dev, "PCIE20_PLR_IATU_LTAR:0x%x\n",

		readl_relaxed(dev->dm_core + PCIE20_PLR_IATU_LTAR));

	PCIE_DBG(dev, "PCIE20_PLR_IATU_UTAR:0x%x\n",

		readl_relaxed(dev->dm_core + PCIE20_PLR_IATU_UTAR));

	PCIE_DBG(dev, "PCIE20_PLR_IATU_CTRL2:0x%x\n",

		readl_relaxed(dev->dm_core + PCIE20_PLR_IATU_CTRL2));

	if (dev->enumerated) {

		msm_pcie_iatu_config_all_ep(dev);

	} else {

		dev->current_bdf = 0; /* to force IATU re-config */

		msm_pcie_cfg_bdf(dev, 1, 0);

	}



	/* configure N_FTS */

	PCIE_DBG(dev, "Original PCIE20_ACK_F_ASPM_CTRL_REG:0x%x\n",
@@ -1377,7 +1512,7 @@ static void msm_pcie_config_link_state(struct msm_pcie_dev_t *dev) 
			msm_pcie_write_mask(dev->rc_shadow +

						PCIE20_CAP_LINKCTRLSTATUS / 4,

						0, BIT(0));

			msm_pcie_write_mask(dev->ep_shadow +

			msm_pcie_write_mask(dev->ep_shadow[0] +

						PCIE20_CAP_LINKCTRLSTATUS / 4,

						0, BIT(0));

		}
@@ -1397,7 +1532,7 @@ static void msm_pcie_config_link_state(struct msm_pcie_dev_t *dev) 
			msm_pcie_write_mask(dev->rc_shadow +

						PCIE20_CAP_LINKCTRLSTATUS / 4,

						0, BIT(1));

			msm_pcie_write_mask(dev->ep_shadow +

			msm_pcie_write_mask(dev->ep_shadow[0] +

						PCIE20_CAP_LINKCTRLSTATUS / 4,

						0, BIT(1));

		}
@@ -1426,11 +1561,11 @@ static void msm_pcie_config_link_state(struct msm_pcie_dev_t *dev) 
			msm_pcie_write_mask(dev->rc_shadow +

					PCIE20_DEVICE_CONTROL2_STATUS2 / 4,

					0, BIT(10));

			msm_pcie_write_mask(dev->ep_shadow +

			msm_pcie_write_mask(dev->ep_shadow[0] +

					PCIE20_L1SUB_CONTROL1 / 4 +

					PCIE20_EP_L1SUB_CTL1_OFFSET / 4, 0,

					BIT(3)|BIT(2)|BIT(1)|BIT(0));

			msm_pcie_write_mask(dev->ep_shadow +

			msm_pcie_write_mask(dev->ep_shadow[0] +

					PCIE20_DEVICE_CONTROL2_STATUS2 / 4,

					0, BIT(10));

		}
@@ -2003,9 +2138,109 @@ static struct hw_pci msm_pci[MAX_RC_NUM] = { 
	},

};



int msm_pcie_enumerate(u32 rc_idx)

static int msm_pcie_config_device_table(struct device *dev, void *pdev)

{

	struct pci_dev *pcidev = to_pci_dev(dev);

	struct msm_pcie_dev_t *pcie_dev = (struct msm_pcie_dev_t *) pdev;

	struct msm_pcie_device_info *dev_table_t = pcie_dev->pcidev_table;

	struct resource *axi_conf = pcie_dev->res[MSM_PCIE_RES_CONF].resource;

	int ret = 0;

	u32 rc_idx = pcie_dev->rc_idx;

	u32 i, index;

	u32 bdf = 0;

	u8 type;

	u32 h_type;

	u32 bme;



	if (!pcidev) {

		PCIE_ERR(pcie_dev,

			"PCIe: Did not find PCI device in list for RC%d.\n",

			pcie_dev->rc_idx);

		return -ENODEV;

	} else {

		PCIE_DBG(pcie_dev,

			"PCI device found: vendor-id:0x%x device-id:0x%x\n",

			pcidev->vendor, pcidev->device);

	}



	if (!pcidev->bus->number)

		return ret;



	bdf = BDF_OFFSET(pcidev->bus->number, pcidev->devfn);

	type = pcidev->bus->number == 1 ?

		PCIE20_CTRL1_TYPE_CFG0 : PCIE20_CTRL1_TYPE_CFG1;



	for (i = 0; i < (MAX_RC_NUM * MAX_DEVICE_NUM); i++) {

		if (msm_pcie_dev_tbl[i].bdf == bdf &&

			!msm_pcie_dev_tbl[i].dev) {

			for (index = 0; index < MAX_DEVICE_NUM; index++) {

				if (dev_table_t[index].bdf == bdf) {

					msm_pcie_dev_tbl[i].dev = pcidev;

					msm_pcie_dev_tbl[i].domain = rc_idx;

					msm_pcie_dev_tbl[i].conf_base =

						pcie_dev->conf + index * SZ_4K;

					msm_pcie_dev_tbl[i].phy_address =

						axi_conf->start + index * SZ_4K;



					dev_table_t[index].dev = pcidev;

					dev_table_t[index].domain = rc_idx;

					dev_table_t[index].conf_base =

						pcie_dev->conf + index * SZ_4K;

					dev_table_t[index].phy_address =

						axi_conf->start + index * SZ_4K;



					msm_pcie_iatu_config(pcie_dev, index,

						type,

						dev_table_t[index].phy_address,

						dev_table_t[index].phy_address

						+ SZ_4K - 1,

						bdf);



					h_type = readl_relaxed(

						dev_table_t[index].conf_base +

						PCIE20_HEADER_TYPE);



					bme = readl_relaxed(

						dev_table_t[index].conf_base +

						PCIE20_COMMAND_STATUS);



					if (h_type & (1 << 16)) {

						pci_write_config_dword(pcidev,

							PCIE20_COMMAND_STATUS,

							bme | 0x06);

					}

					break;

				}

			}

			if (index == MAX_DEVICE_NUM) {

				PCIE_ERR(pcie_dev,

					"RC%d PCI device table is full.\n",

					rc_idx);

				ret = index;

			} else {

				break;

			}

		} else if (msm_pcie_dev_tbl[i].bdf == bdf &&

			pcidev == msm_pcie_dev_tbl[i].dev) {

			break;

		}

	}

	if (i == MAX_RC_NUM * MAX_DEVICE_NUM) {

		PCIE_ERR(pcie_dev,

			"Global PCI device table is full: %d elements.\n",

			i);

		PCIE_ERR(pcie_dev,

			"Bus number is 0x%x\nDevice number is 0x%x\n",

			pcidev->bus->number, pcidev->devfn);

		ret = i;

	}

	return ret;

}





int msm_pcie_enumerate(u32 rc_idx)

{

	int ret = 0, bus_ret = 0;

	struct msm_pcie_dev_t *dev = &msm_pcie_dev[rc_idx];



	PCIE_DBG(dev, "Enumerate RC%d\n", rc_idx);
@@ -2067,6 +2302,16 @@ int msm_pcie_enumerate(u32 rc_idx) 
					dev->rc_idx);

				return -ENODEV;

			}



			bus_ret = bus_for_each_dev(&pci_bus_type, NULL, dev,

					&msm_pcie_config_device_table);



			if (bus_ret) {

				PCIE_ERR(dev,

					"PCIe: Failed to set up device table for RC%d\n",

					dev->rc_idx);

				return -ENODEV;

			}

		} else {

			PCIE_ERR(dev, "PCIe: failed to enable RC%d.\n",

				dev->rc_idx);
@@ -2600,7 +2845,7 @@ static int msm_pcie_probe(struct platform_device *pdev) 
{

	int ret = 0;

	int rc_idx = -1;

	int i;

	int i, j;



	PCIE_GEN_DBG("%s\n", __func__);
@@ -2739,9 +2984,17 @@ static int msm_pcie_probe(struct platform_device *pdev) 
	memcpy(msm_pcie_dev[rc_idx].irq, msm_pcie_irq_info,

				sizeof(msm_pcie_irq_info));

	msm_pcie_dev[rc_idx].shadow_en = true;

	for (i = 0; i < PCIE_CONF_SPACE_DW; i++) {

	for (i = 0; i < PCIE_CONF_SPACE_DW; i++)

		msm_pcie_dev[rc_idx].rc_shadow[i] = PCIE_CLEAR;

		msm_pcie_dev[rc_idx].ep_shadow[i] = PCIE_CLEAR;

	for (i = 0; i < MAX_DEVICE_NUM; i++)

		for (j = 0; j < PCIE_CONF_SPACE_DW; j++)

			msm_pcie_dev[rc_idx].ep_shadow[i][j] = PCIE_CLEAR;

	for (i = 0; i < MAX_DEVICE_NUM; i++) {

		msm_pcie_dev[rc_idx].pcidev_table[i].bdf = 0;

		msm_pcie_dev[rc_idx].pcidev_table[i].dev = NULL;

		msm_pcie_dev[rc_idx].pcidev_table[i].domain = rc_idx;

		msm_pcie_dev[rc_idx].pcidev_table[i].conf_base = 0;

		msm_pcie_dev[rc_idx].pcidev_table[i].phy_address = 0;

	}



	ret = msm_pcie_get_resources(&msm_pcie_dev[rc_idx],
@@ -2876,6 +3129,13 @@ int __init pcie_init(void) 
		spin_lock_init(&msm_pcie_dev[i].wakeup_lock);

		msm_pcie_dev[i].drv_ready = false;

	}

	for (i = 0; i < MAX_RC_NUM * MAX_DEVICE_NUM; i++) {

		msm_pcie_dev_tbl[i].bdf = 0;

		msm_pcie_dev_tbl[i].dev = NULL;

		msm_pcie_dev_tbl[i].domain = -1;

		msm_pcie_dev_tbl[i].conf_base = 0;

		msm_pcie_dev_tbl[i].phy_address = 0;

	}



	ret = platform_driver_register(&msm_pcie_driver);