Merge "msm: pcie: improve link recovery mechanism" (1744d76a) · Commits · e / devices / android_kernel_sony_msm8994

arch/arm/mach-msm/pcie.c

+29 −85

Original line number	Original line	Diff line number	Diff line
	@@ -1356,7 +1356,6 @@ void msm_pcie_disable(struct msm_pcie_dev_t *dev, u32 options)
	dev->gpio[MSM_PCIE_GPIO_PERST].on);		dev->gpio[MSM_PCIE_GPIO_PERST].on);

	if (options & PM_CLK) {		if (options & PM_CLK) {
	if (!(options & PM_EXPT))
	msm_pcie_write_mask(dev->parf + PCIE20_PARF_PHY_CTRL, 0,		msm_pcie_write_mask(dev->parf + PCIE20_PARF_PHY_CTRL, 0,
	BIT(0));		BIT(0));
	msm_pcie_clk_deinit(dev);		msm_pcie_clk_deinit(dev);
	@@ -1641,8 +1640,6 @@ static int msm_pcie_probe(struct platform_device *pdev)
	msm_pcie_dev[rc_idx].user_suspend = false;		msm_pcie_dev[rc_idx].user_suspend = false;
	msm_pcie_dev[rc_idx].saved_state = NULL;		msm_pcie_dev[rc_idx].saved_state = NULL;
	msm_pcie_dev[rc_idx].enumerated = false;		msm_pcie_dev[rc_idx].enumerated = false;
	msm_pcie_dev[rc_idx].handling_linkdown = 0;
	msm_pcie_dev[rc_idx].recovery_pending = false;
	msm_pcie_dev[rc_idx].linkdown_counter = 0;		msm_pcie_dev[rc_idx].linkdown_counter = 0;
	msm_pcie_dev[rc_idx].suspending = false;		msm_pcie_dev[rc_idx].suspending = false;
	msm_pcie_dev[rc_idx].wake_counter = 0;		msm_pcie_dev[rc_idx].wake_counter = 0;
	@@ -1792,6 +1789,8 @@ static int __init pcie_init(void)
	msm_pcie_dev[i].cfg_access = true;		msm_pcie_dev[i].cfg_access = true;
	mutex_init(&msm_pcie_dev[i].setup_lock);		mutex_init(&msm_pcie_dev[i].setup_lock);
	mutex_init(&msm_pcie_dev[i].recovery_lock);		mutex_init(&msm_pcie_dev[i].recovery_lock);
			spin_lock_init(&msm_pcie_dev[i].linkdown_lock);
			spin_lock_init(&msm_pcie_dev[i].wakeup_lock);
	}		}

	ret = platform_driver_register(&msm_pcie_driver);		ret = platform_driver_register(&msm_pcie_driver);
	@@ -1840,7 +1839,8 @@ static int msm_pcie_pm_suspend(struct pci_dev *dev,
	return ret;		return ret;
	}		}

	if (dev && !(options & MSM_PCIE_CONFIG_NO_CFG_RESTORE)) {		if (dev && !(options & MSM_PCIE_CONFIG_NO_CFG_RESTORE)
			&& msm_pcie_confirm_linkup(pcie_dev, true, true)) {
	ret = pci_save_state(dev);		ret = pci_save_state(dev);
	pcie_dev->saved_state = pci_store_saved_state(dev);		pcie_dev->saved_state = pci_store_saved_state(dev);
	}		}
	@@ -1874,10 +1874,6 @@ static int msm_pcie_pm_suspend(struct pci_dev *dev,
	PCIE_DBG(pcie_dev, "RC%d: PM_Enter_L23 is NOT received\n",		PCIE_DBG(pcie_dev, "RC%d: PM_Enter_L23 is NOT received\n",
	pcie_dev->rc_idx);		pcie_dev->rc_idx);

	if (options & MSM_PCIE_CONFIG_LINKDOWN)
	msm_pcie_disable(pcie_dev, PM_EXPT \| PM_PIPE_CLK \|
	PM_CLK \| PM_VREG);
	else
	msm_pcie_disable(pcie_dev, PM_PIPE_CLK \| PM_CLK \| PM_VREG);		msm_pcie_disable(pcie_dev, PM_PIPE_CLK \| PM_CLK \| PM_VREG);

	return ret;		return ret;
	@@ -1953,43 +1949,13 @@ void msm_pcie_fixup_resume(struct pci_dev *dev)
	pcie_dev->user_suspend)		pcie_dev->user_suspend)
	return;		return;

	if (pcie_dev->recovery_pending) {
	PCIE_DBG(pcie_dev,
	"RC%d is pending recovery; so ignore resume.\n",
	pcie_dev->rc_idx);
	return;
	}

	mutex_lock(&pcie_dev->recovery_lock);		mutex_lock(&pcie_dev->recovery_lock);
	ret = msm_pcie_pm_resume(dev, NULL, NULL, 0);		ret = msm_pcie_pm_resume(dev, NULL, NULL, 0);
	if (ret) {		if (ret)
	PCIE_ERR(pcie_dev,		PCIE_ERR(pcie_dev,
	"PCIe: RC%d got failure in fixup resume:%d.\n",		"PCIe: RC%d got failure in fixup resume:%d.\n",
	pcie_dev->rc_idx, ret);		pcie_dev->rc_idx, ret);

	if (pcie_dev->event_reg && pcie_dev->event_reg->callback &&
	(pcie_dev->event_reg->events &
	MSM_PCIE_EVENT_WAKE_RECOVERY)) {
	pcie_dev->recovery_pending = true;
	PCIE_DBG(pcie_dev,
	"PCIe: wait for wake IRQ to recover the link for RC%d\n",
	pcie_dev->rc_idx);
	}

	if (pcie_dev->event_reg && pcie_dev->event_reg->callback &&
	(pcie_dev->event_reg->events &
	MSM_PCIE_EVENT_NO_ACCESS)) {
	struct msm_pcie_notify *notify =
	&pcie_dev->event_reg->notify;
	notify->event = MSM_PCIE_EVENT_NO_ACCESS;
	notify->user = pcie_dev->event_reg->user;
	PCIE_DBG(pcie_dev,
	"PCIe: notify client not to access the EP of RC%d\n",
	pcie_dev->rc_idx);
	pcie_dev->event_reg->callback(notify);
	}
	}

	mutex_unlock(&pcie_dev->recovery_lock);		mutex_unlock(&pcie_dev->recovery_lock);
	}		}
	DECLARE_PCI_FIXUP_RESUME(PCIE_VENDOR_ID_RCP, PCIE_DEVICE_ID_RCP,		DECLARE_PCI_FIXUP_RESUME(PCIE_VENDOR_ID_RCP, PCIE_DEVICE_ID_RCP,
	@@ -2006,41 +1972,12 @@ void msm_pcie_fixup_resume_early(struct pci_dev *dev)
	pcie_dev->user_suspend)		pcie_dev->user_suspend)
	return;		return;

	if (pcie_dev->recovery_pending) {
	PCIE_DBG(pcie_dev,
	"RC%d is pending recovery; so ignore resume.\n",
	pcie_dev->rc_idx);
	return;
	}

	mutex_lock(&pcie_dev->recovery_lock);		mutex_lock(&pcie_dev->recovery_lock);
	ret = msm_pcie_pm_resume(dev, NULL, NULL, 0);		ret = msm_pcie_pm_resume(dev, NULL, NULL, 0);
	if (ret) {		if (ret)
	PCIE_ERR(pcie_dev, "PCIe: RC%d got failure in resume:%d.\n",		PCIE_ERR(pcie_dev, "PCIe: RC%d got failure in resume:%d.\n",
	pcie_dev->rc_idx, ret);		pcie_dev->rc_idx, ret);

	if (pcie_dev->event_reg && pcie_dev->event_reg->callback &&
	(pcie_dev->event_reg->events &
	MSM_PCIE_EVENT_WAKE_RECOVERY)) {
	pcie_dev->recovery_pending = true;
	PCIE_DBG(pcie_dev,
	"PCIe: wait for wake IRQ to recover the link for RC%d\n",
	pcie_dev->rc_idx);
	}

	if (pcie_dev->event_reg && pcie_dev->event_reg->callback &&
	(pcie_dev->event_reg->events &
	MSM_PCIE_EVENT_NO_ACCESS)) {
	struct msm_pcie_notify *notify =
	&pcie_dev->event_reg->notify;
	notify->event = MSM_PCIE_EVENT_NO_ACCESS;
	notify->user = pcie_dev->event_reg->user;
	PCIE_DBG(pcie_dev,
	"PCIe: notify client not to access the EP of RC%d\n",
	pcie_dev->rc_idx);
	pcie_dev->event_reg->callback(notify);
	}
	}
	mutex_unlock(&pcie_dev->recovery_lock);		mutex_unlock(&pcie_dev->recovery_lock);
	}		}
	DECLARE_PCI_FIXUP_RESUME_EARLY(PCIE_VENDOR_ID_RCP, PCIE_DEVICE_ID_RCP,		DECLARE_PCI_FIXUP_RESUME_EARLY(PCIE_VENDOR_ID_RCP, PCIE_DEVICE_ID_RCP,
	@@ -2094,17 +2031,22 @@ int msm_pcie_pm_control(enum msm_pcie_pm_opt pm_opt, u32 busnr, void *user,

	switch (pm_opt) {		switch (pm_opt) {
	case MSM_PCIE_SUSPEND:		case MSM_PCIE_SUSPEND:
	if ((msm_pcie_dev[rc_idx].link_status != MSM_PCIE_LINK_ENABLED)		if (msm_pcie_dev[rc_idx].link_status != MSM_PCIE_LINK_ENABLED)
	&& !(options & MSM_PCIE_CONFIG_LINKDOWN)) {		PCIE_DBG(&msm_pcie_dev[rc_idx],
	PCIE_ERR(&msm_pcie_dev[rc_idx],
	"PCIe: RC%d: requested to suspend when link is not enabled:%d.\n",		"PCIe: RC%d: requested to suspend when link is not enabled:%d.\n",
	rc_idx, msm_pcie_dev[rc_idx].link_status);		rc_idx, msm_pcie_dev[rc_idx].link_status);

			if (!msm_pcie_dev[rc_idx].power_on) {
			PCIE_ERR(&msm_pcie_dev[rc_idx],
			"PCIe: RC%d: requested to suspend when link is powered down:%d.\n",
			rc_idx, msm_pcie_dev[rc_idx].link_status);
	break;		break;
	}		}
	if (!(options & MSM_PCIE_CONFIG_LINKDOWN)) {
	msm_pcie_dev[rc_idx].user_suspend = true;		msm_pcie_dev[rc_idx].user_suspend = true;

	mutex_lock(&msm_pcie_dev[rc_idx].recovery_lock);		mutex_lock(&msm_pcie_dev[rc_idx].recovery_lock);
	}
	ret = msm_pcie_pm_suspend(dev, user, data, options);		ret = msm_pcie_pm_suspend(dev, user, data, options);
	if (ret) {		if (ret) {
	PCIE_ERR(&msm_pcie_dev[rc_idx],		PCIE_ERR(&msm_pcie_dev[rc_idx],
	@@ -2112,7 +2054,7 @@ int msm_pcie_pm_control(enum msm_pcie_pm_opt pm_opt, u32 busnr, void *user,
	rc_idx);		rc_idx);
	msm_pcie_dev[rc_idx].user_suspend = false;		msm_pcie_dev[rc_idx].user_suspend = false;
	}		}
	if (!(options & MSM_PCIE_CONFIG_LINKDOWN))
	mutex_unlock(&msm_pcie_dev[rc_idx].recovery_lock);		mutex_unlock(&msm_pcie_dev[rc_idx].recovery_lock);
	break;		break;
	case MSM_PCIE_RESUME:		case MSM_PCIE_RESUME:
	@@ -2125,7 +2067,7 @@ int msm_pcie_pm_control(enum msm_pcie_pm_opt pm_opt, u32 busnr, void *user,
	rc_idx, msm_pcie_dev[rc_idx].link_status);		rc_idx, msm_pcie_dev[rc_idx].link_status);
	break;		break;
	}		}
	if (!(options & MSM_PCIE_CONFIG_LINKDOWN))
	mutex_lock(&msm_pcie_dev[rc_idx].recovery_lock);		mutex_lock(&msm_pcie_dev[rc_idx].recovery_lock);
	ret = msm_pcie_pm_resume(dev, user, data, options);		ret = msm_pcie_pm_resume(dev, user, data, options);
	if (ret) {		if (ret) {
	@@ -2133,11 +2075,13 @@ int msm_pcie_pm_control(enum msm_pcie_pm_opt pm_opt, u32 busnr, void *user,
	"PCIe: RC%d: user failed to resume the link.\n",		"PCIe: RC%d: user failed to resume the link.\n",
	rc_idx);		rc_idx);
	} else {		} else {
			PCIE_DBG(&msm_pcie_dev[rc_idx],
			"PCIe: RC%d: user succeeded to resume the link.\n",
			rc_idx);

	msm_pcie_dev[rc_idx].user_suspend = false;		msm_pcie_dev[rc_idx].user_suspend = false;
	msm_pcie_dev[rc_idx].recovery_pending = false;
	}		}

	if (!(options & MSM_PCIE_CONFIG_LINKDOWN))
	mutex_unlock(&msm_pcie_dev[rc_idx].recovery_lock);		mutex_unlock(&msm_pcie_dev[rc_idx].recovery_lock);
	break;		break;
	default:		default:

arch/arm/mach-msm/pcie.h

+2 −4

Original line number	Original line	Diff line number	Diff line
	@@ -67,7 +67,6 @@
	#define PM_GPIO 0x4		#define PM_GPIO 0x4
	#define PM_VREG 0x8		#define PM_VREG 0x8
	#define PM_PIPE_CLK 0x10		#define PM_PIPE_CLK 0x10
	#define PM_EXPT 0x80000000
	#define PM_ALL (PM_IRQ \| PM_CLK \| PM_GPIO \| PM_VREG \| PM_PIPE_CLK)		#define PM_ALL (PM_IRQ \| PM_CLK \| PM_GPIO \| PM_VREG \| PM_PIPE_CLK)

	#define PCIE_CONF_SPACE_DW 1024		#define PCIE_CONF_SPACE_DW 1024
	@@ -216,10 +215,9 @@ struct msm_pcie_dev_t {
	uint32_t rc_idx;		uint32_t rc_idx;
	bool enumerated;		bool enumerated;
	struct work_struct handle_wake_work;		struct work_struct handle_wake_work;
	struct work_struct handle_linkdown_work;
	int handling_linkdown;
	bool recovery_pending;
	struct mutex recovery_lock;		struct mutex recovery_lock;
			spinlock_t linkdown_lock;
			spinlock_t wakeup_lock;
	ulong linkdown_counter;		ulong linkdown_counter;
	bool suspending;		bool suspending;
	ulong wake_counter;		ulong wake_counter;

arch/arm/mach-msm/pcie_irq.c

+40 −206

Original line number	Original line	Diff line number	Diff line
	@@ -45,93 +45,30 @@
	#define LINKDOWN_WAITING_US_MAX 5100		#define LINKDOWN_WAITING_US_MAX 5100
	#define LINKDOWN_WAITING_COUNT 200		#define LINKDOWN_WAITING_COUNT 200

	static int msm_pcie_recover_link(struct msm_pcie_dev_t *dev)		static void msm_pcie_notify_client(struct msm_pcie_dev_t *dev,
	{		enum msm_pcie_event event)
	int ret;

	ret = msm_pcie_enable(dev, PM_PIPE_CLK \| PM_CLK \| PM_VREG);

	if (!ret) {
	PCIE_DBG(dev, "Recover config space of RC%d and its EP\n",
	dev->rc_idx);
	PCIE_DBG(dev, "Recover RC%d\n", dev->rc_idx);
	msm_pcie_cfg_recover(dev, true);
	PCIE_DBG(dev, "Recover EP of RC%d\n", dev->rc_idx);
	msm_pcie_cfg_recover(dev, false);
	dev->shadow_en = true;

	if ((dev->link_status == MSM_PCIE_LINK_ENABLED) &&
	dev->event_reg && dev->event_reg->callback &&
	(dev->event_reg->events & MSM_PCIE_EVENT_LINKUP)) {
	struct msm_pcie_notify *notify =
	&dev->event_reg->notify;
	notify->event = MSM_PCIE_EVENT_LINKUP;
	notify->user = dev->event_reg->user;
	PCIE_DBG(dev, "Linkup callback for RC%d\n",
	dev->rc_idx);
	dev->event_reg->callback(notify);
	}
	}

	return ret;
	}

	static void msm_pcie_notify_linkdown(struct msm_pcie_dev_t *dev)
	{		{
	if (dev->event_reg && dev->event_reg->callback &&		if (dev->event_reg && dev->event_reg->callback &&
	(dev->event_reg->events & MSM_PCIE_EVENT_LINKDOWN)) {		(dev->event_reg->events & event)) {
	struct msm_pcie_notify *notify = &dev->event_reg->notify;		struct msm_pcie_notify *notify = &dev->event_reg->notify;
	notify->event = MSM_PCIE_EVENT_LINKDOWN;		notify->event = event;
	notify->user = dev->event_reg->user;		notify->user = dev->event_reg->user;
	PCIE_DBG(dev, "PCIe: Linkdown callback for RC%d\n",		PCIE_DBG(dev, "PCIe: callback RC%d for event %d.\n",
	dev->rc_idx);		dev->rc_idx, event);
	dev->event_reg->callback(notify);		dev->event_reg->callback(notify);

	if (dev->event_reg->options & MSM_PCIE_CONFIG_NO_RECOVERY) {		if ((dev->event_reg->options & MSM_PCIE_CONFIG_NO_RECOVERY) &&
			(event == MSM_PCIE_EVENT_LINKDOWN)) {
	dev->user_suspend = true;		dev->user_suspend = true;
	PCIE_DBG(dev,		PCIE_DBG(dev,
	"PCIe: Client of RC%d will recover the link later.\n",		"PCIe: Client of RC%d will recover the link later.\n",
	dev->rc_idx);		dev->rc_idx);
	return;		return;
	}		}

	if (dev->link_status == MSM_PCIE_LINK_DISABLED) {
	PCIE_DBG(dev,
	"PCIe: Client of RC%d does not enable link in callback; so disable the link\n",
	dev->rc_idx);
	dev->recovery_pending = true;
	msm_pcie_disable(dev,
	PM_EXPT \| PM_PIPE_CLK \| PM_CLK \| PM_VREG);
	} else {		} else {
	dev->recovery_pending = false;
	PCIE_DBG(dev,
	"PCIe: Client of RC%d has enabled link in callback; so recover config space\n",
	dev->rc_idx);
	PCIE_DBG(dev, "PCIe: Recover RC%d\n", dev->rc_idx);
	msm_pcie_cfg_recover(dev, true);
	PCIE_DBG(dev, "PCIe: Recover EP of RC%d\n",
	dev->rc_idx);
	msm_pcie_cfg_recover(dev, false);
	dev->shadow_en = true;

	if ((dev->link_status == MSM_PCIE_LINK_ENABLED) &&
	dev->event_reg && dev->event_reg->callback &&
	(dev->event_reg->events &
	MSM_PCIE_EVENT_LINKUP)) {
	struct msm_pcie_notify *notify =
	&dev->event_reg->notify;
	notify->event = MSM_PCIE_EVENT_LINKUP;
	notify->user = dev->event_reg->user;
	PCIE_DBG(dev,		PCIE_DBG(dev,
	"PCIe: Linkup callback for RC%d\n",		"PCIe: Client of RC%d does not have registration for event %d.\n",
	dev->rc_idx);		dev->rc_idx, event);
	dev->event_reg->callback(notify);
	}
	}
	} else {
	PCIE_ERR(dev,
	"PCIe: Client driver does not have registration and this linkdown of RC%d should never happen.\n",
	dev->rc_idx);
	}		}
	}		}

	@@ -143,15 +80,20 @@ static void handle_wake_func(struct work_struct *work)

	PCIE_DBG(dev, "PCIe: Wake work for RC%d\n", dev->rc_idx);		PCIE_DBG(dev, "PCIe: Wake work for RC%d\n", dev->rc_idx);

	if (!dev->enumerated) {
	mutex_lock(&dev->recovery_lock);		mutex_lock(&dev->recovery_lock);

			if (!dev->enumerated) {
			PCIE_DBG(dev,
			"PCIe: Start enumeration for RC%d upon the wake from endpoint.\n",
			dev->rc_idx);

	ret = msm_pcie_enumerate(dev->rc_idx);		ret = msm_pcie_enumerate(dev->rc_idx);
	mutex_unlock(&dev->recovery_lock);
	if (ret) {		if (ret) {
	PCIE_ERR(dev,		PCIE_ERR(dev,
	"PCIe: failed to enable RC%d upon wake request from the device.\n",		"PCIe: failed to enable RC%d upon wake request from the device.\n",
	dev->rc_idx);		dev->rc_idx);
	return;		goto out;
	}		}

	if ((dev->link_status == MSM_PCIE_LINK_ENABLED) &&		if ((dev->link_status == MSM_PCIE_LINK_ENABLED) &&
	@@ -165,109 +107,18 @@ static void handle_wake_func(struct work_struct *work)
	"PCIe: Linkup callback for RC%d after enumeration is successful in wake IRQ handling\n",		"PCIe: Linkup callback for RC%d after enumeration is successful in wake IRQ handling\n",
	dev->rc_idx);		dev->rc_idx);
	dev->event_reg->callback(notify);		dev->event_reg->callback(notify);
	}
	return;
	} else {		} else {
	int waiting_cycle = 0;
	usleep_range(LINKDOWN_INIT_WAITING_US_MIN,
	LINKDOWN_INIT_WAITING_US_MAX);
	while ((dev->handling_linkdown > 0) &&
	(waiting_cycle++ < LINKDOWN_WAITING_COUNT)) {
	usleep_range(LINKDOWN_WAITING_US_MIN,
	LINKDOWN_WAITING_US_MAX);
	}

	if (waiting_cycle == LINKDOWN_WAITING_COUNT)
	PCIE_ERR(dev,
	"PCIe: Linkdown handling for RC%d is not finished after max waiting time.\n",
	dev->rc_idx);

	mutex_lock(&dev->recovery_lock);
	if (dev->link_status == MSM_PCIE_LINK_ENABLED) {
	PCIE_DBG(dev,		PCIE_DBG(dev,
	"PCIe: The link status of RC%d is up. Check if it is really up.\n",		"PCIe: Client of RC%d does not have registration for linkup event.\n",
	dev->rc_idx);		dev->rc_idx);

	if (msm_pcie_confirm_linkup(dev, false, true)) {
	PCIE_DBG(dev,
	"PCIe: The link status of RC%d is really up; so ignore wake IRQ.\n",
	dev->rc_idx);
	goto out;
	} else {
	dev->link_status = MSM_PCIE_LINK_DISABLED;
	dev->shadow_en = false;
	/* assert PERST */
	gpio_set_value(
	dev->gpio[MSM_PCIE_GPIO_PERST].num,
	dev->gpio[MSM_PCIE_GPIO_PERST].on);
	PCIE_ERR(dev,
	"PCIe: The link of RC%d is actually down; notify the client.\n",
	dev->rc_idx);

	msm_pcie_notify_linkdown(dev);
	}		}
	} else {
	PCIE_DBG(dev,
	"PCIe: The link status of RC%d is down.\n",
	dev->rc_idx);

	if (dev->recovery_pending) {
	static u32 retries = 1;
	PCIE_DBG(dev,
	"PCIe: Start recovering link for RC%d after receive wake IRQ.\n",
	dev->rc_idx);
	ret = msm_pcie_recover_link(dev);
	if (ret) {
	PCIE_ERR(dev,
	"PCIe:failed to enable link for RC%d in No. %d try after receive wake IRQ.\n",
	dev->rc_idx, retries++);
	goto out;		goto out;
	} else {
	dev->recovery_pending = false;
	PCIE_DBG(dev,
	"PCIe: Successful recovery for RC%d in No. %d try.\n",
	dev->rc_idx, retries);
	retries = 1;
	}
	} else if (dev->user_suspend) {
	PCIE_DBG(dev,
	"PCIe: wake IRQ for RC%d for a user-suspended link.\n",
	dev->rc_idx);
	if (dev->event_reg &&
	dev->event_reg->callback &&
	(dev->event_reg->events &
	MSM_PCIE_EVENT_WAKEUP)) {
	struct msm_pcie_notify *nfy =
	&dev->event_reg->notify;
	nfy->event = MSM_PCIE_EVENT_WAKEUP;
	nfy->user = dev->event_reg->user;
	PCIE_DBG(dev,
	"PCIe: wakeup callback for RC%d\n",
	dev->rc_idx);
	dev->event_reg->callback(nfy);
	if (dev->link_status ==
	MSM_PCIE_LINK_ENABLED)
	PCIE_DBG(dev,
	"PCIe: link is enabled after wakeup callback for RC%d\n",
	dev->rc_idx);
	else
	PCIE_DBG(dev,
	"PCIe: link is NOT enabled after wakeup callback for RC%d\n",
	dev->rc_idx);
	} else {		} else {
	PCIE_ERR(dev,		PCIE_ERR(dev,
	"PCIe: client of RC%d does not register callback for wake IRQ for a user-suspended link.\n",		"PCIe: The enumeration for RC%d has already been done.\n",
	dev->rc_idx);
	}
	goto out;
	} else {
	PCIE_DBG(dev,
	"PCIe: No pending recovery or user-issued suspend for RC%d; so ignore wake IRQ.\n",
	dev->rc_idx);		dev->rc_idx);
	goto out;		goto out;
	}		}
	}
	}

	out:		out:
	mutex_unlock(&dev->recovery_lock);		mutex_unlock(&dev->recovery_lock);
	@@ -276,6 +127,9 @@ out:
	static irqreturn_t handle_wake_irq(int irq, void *data)		static irqreturn_t handle_wake_irq(int irq, void *data)
	{		{
	struct msm_pcie_dev_t *dev = data;		struct msm_pcie_dev_t *dev = data;
			unsigned long irqsave_flags;

			spin_lock_irqsave(&dev->wakeup_lock, irqsave_flags);

	dev->wake_counter++;		dev->wake_counter++;
	PCIE_DBG(dev, "PCIe: No. %ld wake IRQ for RC%d\n",		PCIE_DBG(dev, "PCIe: No. %ld wake IRQ for RC%d\n",
	@@ -291,45 +145,25 @@ static irqreturn_t handle_wake_irq(int irq, void *data)
	PCIE_DBG(dev, "Wake up RC%d\n", dev->rc_idx);		PCIE_DBG(dev, "Wake up RC%d\n", dev->rc_idx);
	__pm_stay_awake(&dev->ws);		__pm_stay_awake(&dev->ws);
	__pm_relax(&dev->ws);		__pm_relax(&dev->ws);
			msm_pcie_notify_client(dev, MSM_PCIE_EVENT_WAKEUP);
	schedule_work(&dev->handle_wake_work);
	}

	return IRQ_HANDLED;
	}		}

	static void handle_linkdown_func(struct work_struct *work)		spin_unlock_irqrestore(&dev->wakeup_lock, irqsave_flags);
	{
	struct msm_pcie_dev_t *dev = container_of(work, struct msm_pcie_dev_t,
	handle_linkdown_work);

	PCIE_DBG(dev, "PCIe: Linkdown work for RC%d\n", dev->rc_idx);

	mutex_lock(&dev->recovery_lock);		return IRQ_HANDLED;

	if (msm_pcie_confirm_linkup(dev, true, true))
	PCIE_DBG(dev,
	"PCIe: The link status of RC%d is up now, indicating recovery has been done.\n",
	dev->rc_idx);
	else
	msm_pcie_notify_linkdown(dev);

	dev->handling_linkdown--;
	if (dev->handling_linkdown < 0)
	PCIE_ERR(dev, "PCIe:handling_linkdown for RC%d is %d\n",
	dev->rc_idx, dev->handling_linkdown);
	mutex_unlock(&dev->recovery_lock);
	}		}

	static irqreturn_t handle_linkdown_irq(int irq, void *data)		static irqreturn_t handle_linkdown_irq(int irq, void *data)
	{		{
	struct msm_pcie_dev_t *dev = data;		struct msm_pcie_dev_t *dev = data;
			unsigned long irqsave_flags;

			spin_lock_irqsave(&dev->linkdown_lock, irqsave_flags);

	dev->linkdown_counter++;		dev->linkdown_counter++;
	dev->handling_linkdown++;
	PCIE_DBG(dev,		PCIE_DBG(dev,
	"PCIe: No. %ld linkdown IRQ for RC%d: handling_linkdown:%d\n",		"PCIe: No. %ld linkdown IRQ for RC%d.\n",
	dev->linkdown_counter, dev->rc_idx, dev->handling_linkdown);		dev->linkdown_counter, dev->rc_idx);

	if (!dev->enumerated \|\| dev->link_status != MSM_PCIE_LINK_ENABLED) {		if (!dev->enumerated \|\| dev->link_status != MSM_PCIE_LINK_ENABLED) {
	PCIE_DBG(dev,		PCIE_DBG(dev,
	@@ -346,9 +180,11 @@ static irqreturn_t handle_linkdown_irq(int irq, void *data)
	gpio_set_value(dev->gpio[MSM_PCIE_GPIO_PERST].num,		gpio_set_value(dev->gpio[MSM_PCIE_GPIO_PERST].num,
	dev->gpio[MSM_PCIE_GPIO_PERST].on);		dev->gpio[MSM_PCIE_GPIO_PERST].on);
	PCIE_ERR(dev, "PCIe link is down for RC%d\n", dev->rc_idx);		PCIE_ERR(dev, "PCIe link is down for RC%d\n", dev->rc_idx);
	schedule_work(&dev->handle_linkdown_work);		msm_pcie_notify_client(dev, MSM_PCIE_EVENT_LINKDOWN);
	}		}

			spin_unlock_irqrestore(&dev->linkdown_lock, irqsave_flags);

	return IRQ_HANDLED;		return IRQ_HANDLED;
	}		}

	@@ -683,8 +519,6 @@ int32_t msm_pcie_irq_init(struct msm_pcie_dev_t *dev)
	return rc;		return rc;
	}		}

	INIT_WORK(&dev->handle_linkdown_work, handle_linkdown_func);

	/* register handler for physical MSI interrupt line */		/* register handler for physical MSI interrupt line */
	rc = devm_request_irq(pdev,		rc = devm_request_irq(pdev,
	dev->irq[MSM_PCIE_INT_MSI].num, handle_msi_irq,		dev->irq[MSM_PCIE_INT_MSI].num, handle_msi_irq,