Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 9b3c76f0 authored by Gavin Shan's avatar Gavin Shan Committed by Benjamin Herrenschmidt
Browse files

powerpc/eeh: Handle EEH error based on PE 



The patch reworks the current implementation so that the eeh errors
will be handled basing on PE instead of eeh device.

Signed-off-by: default avatarGavin Shan <shangw@linux.vnet.ibm.com>
Signed-off-by: default avatarBenjamin Herrenschmidt <benh@kernel.crashing.org>
parent 120dc496

arch/powerpc/include/asm/eeh.h

+1 −0
Original line number Diff line number Diff line
@@ -174,6 +174,7 @@ int eeh_rmv_from_parent_pe(struct eeh_dev *edev); 
void *eeh_pe_dev_traverse(struct eeh_pe *root,

		eeh_traverse_func fn, void *flag);

void eeh_pe_restore_bars(struct eeh_pe *pe);

struct pci_bus *eeh_pe_bus_get(struct eeh_pe *pe);



void * __devinit eeh_dev_init(struct device_node *dn, void *data);

void __devinit eeh_dev_phb_init_dynamic(struct pci_controller *phb);

arch/powerpc/include/asm/eeh_event.h

+1 −1
Original line number Diff line number Diff line
@@ -32,7 +32,7 @@ struct eeh_event { 
};



int eeh_send_failure_event(struct eeh_pe *pe);

struct eeh_dev *handle_eeh_events(struct eeh_event *);

void eeh_handle_event(struct eeh_pe *pe);



#endif /* __KERNEL__ */

#endif /* ASM_POWERPC_EEH_EVENT_H */

arch/powerpc/platforms/pseries/eeh_driver.c

+94 −135
Original line number Diff line number Diff line
@@ -116,28 +116,35 @@ static void eeh_enable_irq(struct pci_dev *dev) 


/**

 * eeh_report_error - Report pci error to each device driver

 * @dev: PCI device

 * @data: eeh device

 * @userdata: return value

 * 

 * Report an EEH error to each device driver, collect up and 

 * merge the device driver responses. Cumulative response 

 * passed back in "userdata".

 */

static int eeh_report_error(struct pci_dev *dev, void *userdata)

static void *eeh_report_error(void *data, void *userdata)

{

	struct eeh_dev *edev = (struct eeh_dev *)data;

	struct pci_dev *dev = eeh_dev_to_pci_dev(edev);

	enum pci_ers_result rc, *res = userdata;

	struct pci_driver *driver = dev->driver;



	/* We might not have the associated PCI device,

	 * then we should continue for next one.

	 */

	if (!dev) return NULL;



	dev->error_state = pci_channel_io_frozen;



	if (!driver)

		return 0;

		return NULL;



	eeh_disable_irq(dev);



	if (!driver->err_handler ||

	    !driver->err_handler->error_detected)

		return 0;

		return NULL;



	rc = driver->err_handler->error_detected(dev, pci_channel_io_frozen);
@@ -145,27 +152,31 @@ static int eeh_report_error(struct pci_dev *dev, void *userdata) 
	if (rc == PCI_ERS_RESULT_NEED_RESET) *res = rc;

	if (*res == PCI_ERS_RESULT_NONE) *res = rc;



	return 0;

	return NULL;

}



/**

 * eeh_report_mmio_enabled - Tell drivers that MMIO has been enabled

 * @dev: PCI device

 * @data: eeh device

 * @userdata: return value

 *

 * Tells each device driver that IO ports, MMIO and config space I/O

 * are now enabled. Collects up and merges the device driver responses.

 * Cumulative response passed back in "userdata".

 */

static int eeh_report_mmio_enabled(struct pci_dev *dev, void *userdata)

static void *eeh_report_mmio_enabled(void *data, void *userdata)

{

	struct eeh_dev *edev = (struct eeh_dev *)data;

	struct pci_dev *dev = eeh_dev_to_pci_dev(edev);

	enum pci_ers_result rc, *res = userdata;

	struct pci_driver *driver = dev->driver;

	struct pci_driver *driver;



	if (!dev) return NULL;



	if (!driver ||

	if (!(driver = dev->driver) ||

	    !driver->err_handler ||

	    !driver->err_handler->mmio_enabled)

		return 0;

		return NULL;



	rc = driver->err_handler->mmio_enabled(dev);
@@ -173,12 +184,12 @@ static int eeh_report_mmio_enabled(struct pci_dev *dev, void *userdata) 
	if (rc == PCI_ERS_RESULT_NEED_RESET) *res = rc;

	if (*res == PCI_ERS_RESULT_NONE) *res = rc;



	return 0;

	return NULL;

}



/**

 * eeh_report_reset - Tell device that slot has been reset

 * @dev: PCI device

 * @data: eeh device

 * @userdata: return value

 *

 * This routine must be called while EEH tries to reset particular
@@ -186,13 +197,15 @@ static int eeh_report_mmio_enabled(struct pci_dev *dev, void *userdata) 
 * some actions, usually to save data the driver needs so that the

 * driver can work again while the device is recovered.

 */

static int eeh_report_reset(struct pci_dev *dev, void *userdata)

static void *eeh_report_reset(void *data, void *userdata)

{

	struct eeh_dev *edev = (struct eeh_dev *)data;

	struct pci_dev *dev = eeh_dev_to_pci_dev(edev);

	enum pci_ers_result rc, *res = userdata;

	struct pci_driver *driver = dev->driver;

	struct pci_driver *driver;



	if (!driver)

		return 0;

	if (!dev || !(driver = dev->driver))

		return NULL;



	dev->error_state = pci_channel_io_normal;
@@ -200,7 +213,7 @@ static int eeh_report_reset(struct pci_dev *dev, void *userdata) 


	if (!driver->err_handler ||

	    !driver->err_handler->slot_reset)

		return 0;

		return NULL;



	rc = driver->err_handler->slot_reset(dev);

	if ((*res == PCI_ERS_RESULT_NONE) ||
@@ -208,82 +221,89 @@ static int eeh_report_reset(struct pci_dev *dev, void *userdata) 
	if (*res == PCI_ERS_RESULT_DISCONNECT &&

	     rc == PCI_ERS_RESULT_NEED_RESET) *res = rc;



	return 0;

	return NULL;

}



/**

 * eeh_report_resume - Tell device to resume normal operations

 * @dev: PCI device

 * @data: eeh device

 * @userdata: return value

 *

 * This routine must be called to notify the device driver that it

 * could resume so that the device driver can do some initialization

 * to make the recovered device work again.

 */

static int eeh_report_resume(struct pci_dev *dev, void *userdata)

static void *eeh_report_resume(void *data, void *userdata)

{

	struct pci_driver *driver = dev->driver;

	struct eeh_dev *edev = (struct eeh_dev *)data;

	struct pci_dev *dev = eeh_dev_to_pci_dev(edev);

	struct pci_driver *driver;



	if (!dev) return NULL;



	dev->error_state = pci_channel_io_normal;



	if (!driver)

		return 0;

	if (!(driver = dev->driver))

		return NULL;



	eeh_enable_irq(dev);



	if (!driver->err_handler ||

	    !driver->err_handler->resume)

		return 0;

		return NULL;



	driver->err_handler->resume(dev);



	return 0;

	return NULL;

}



/**

 * eeh_report_failure - Tell device driver that device is dead.

 * @dev: PCI device

 * @data: eeh device

 * @userdata: return value

 *

 * This informs the device driver that the device is permanently

 * dead, and that no further recovery attempts will be made on it.

 */

static int eeh_report_failure(struct pci_dev *dev, void *userdata)

static void *eeh_report_failure(void *data, void *userdata)

{

	struct pci_driver *driver = dev->driver;

	struct eeh_dev *edev = (struct eeh_dev *)data;

	struct pci_dev *dev = eeh_dev_to_pci_dev(edev);

	struct pci_driver *driver;



	if (!dev) return NULL;



	dev->error_state = pci_channel_io_perm_failure;



	if (!driver)

		return 0;

	if (!(driver = dev->driver))

		return NULL;



	eeh_disable_irq(dev);



	if (!driver->err_handler ||

	    !driver->err_handler->error_detected)

		return 0;

		return NULL;



	driver->err_handler->error_detected(dev, pci_channel_io_perm_failure);



	return 0;

	return NULL;

}



/**

 * eeh_reset_device - Perform actual reset of a pci slot

 * @edev: PE associated EEH device

 * @pe: EEH PE

 * @bus: PCI bus corresponding to the isolcated slot

 *

 * This routine must be called to do reset on the indicated PE.

 * During the reset, udev might be invoked because those affected

 * PCI devices will be removed and then added.

 */

static int eeh_reset_device(struct eeh_dev *edev, struct pci_bus *bus)

static int eeh_reset_device(struct eeh_pe *pe, struct pci_bus *bus)

{

	struct device_node *dn;

	int cnt, rc;



	/* pcibios will clear the counter; save the value */

	cnt = edev->freeze_count;

	cnt = pe->freeze_count;



	if (bus)

		pcibios_remove_pci_devices(bus);
@@ -292,25 +312,13 @@ static int eeh_reset_device(struct eeh_dev *edev, struct pci_bus *bus) 
	 * Reconfigure bridges and devices. Don't try to bring the system

	 * up if the reset failed for some reason.

	 */

	rc = eeh_reset_pe(edev);

	rc = eeh_reset_pe(pe);

	if (rc)

		return rc;



	/* Walk over all functions on this device. */

	dn = eeh_dev_to_of_node(edev);

	if (!pcibios_find_pci_bus(dn) && of_node_to_eeh_dev(dn->parent))

		dn = dn->parent->child;



	while (dn) {

		struct eeh_dev *pedev = of_node_to_eeh_dev(dn);



		/* On Power4, always true because eeh_pe_config_addr=0 */

		if (edev->pe_config_addr == pedev->pe_config_addr) {

			eeh_ops->configure_bridge(dn);

			eeh_restore_bars(pedev);

 		}

		dn = dn->sibling;

	}

	/* Restore PE */

	eeh_ops->configure_bridge(pe);

	eeh_pe_restore_bars(pe);



	/* Give the system 5 seconds to finish running the user-space

	 * hotplug shutdown scripts, e.g. ifdown for ethernet.  Yes,
@@ -322,7 +330,7 @@ static int eeh_reset_device(struct eeh_dev *edev, struct pci_bus *bus) 
		ssleep(5);

		pcibios_add_pci_devices(bus);

	}

	edev->freeze_count = cnt;

	pe->freeze_count = cnt;



	return 0;

}
@@ -334,7 +342,7 @@ static int eeh_reset_device(struct eeh_dev *edev, struct pci_bus *bus) 


/**

 * eeh_handle_event - Reset a PCI device after hard lockup.

 * @event: EEH event

 * @pe: EEH PE

 *

 * While PHB detects address or data parity errors on particular PCI

 * slot, the associated PE will be frozen. Besides, DMA's occurring
@@ -349,69 +357,24 @@ static int eeh_reset_device(struct eeh_dev *edev, struct pci_bus *bus) 
 * drivers (which cause a second set of hotplug events to go out to

 * userspace).

 */

struct eeh_dev *handle_eeh_events(struct eeh_event *event)

void eeh_handle_event(struct eeh_pe *pe)

{

	struct device_node *frozen_dn;

	struct eeh_dev *frozen_edev;

	struct pci_bus *frozen_bus;

	int rc = 0;

	enum pci_ers_result result = PCI_ERS_RESULT_NONE;

	const char *location, *pci_str, *drv_str, *bus_pci_str, *bus_drv_str;



	frozen_dn = eeh_find_device_pe(eeh_dev_to_of_node(event->edev));

	if (!frozen_dn) {

		location = of_get_property(eeh_dev_to_of_node(event->edev), "ibm,loc-code", NULL);

		location = location ? location : "unknown";

		printk(KERN_ERR "EEH: Error: Cannot find partition endpoint "

		                "for location=%s pci addr=%s\n",

			location, eeh_pci_name(eeh_dev_to_pci_dev(event->edev)));

		return NULL;

	}



	frozen_bus = pcibios_find_pci_bus(frozen_dn);

	location = of_get_property(frozen_dn, "ibm,loc-code", NULL);

	location = location ? location : "unknown";



	/* There are two different styles for coming up with the PE.

	 * In the old style, it was the highest EEH-capable device

	 * which was always an EADS pci bridge.  In the new style,

	 * there might not be any EADS bridges, and even when there are,

	 * the firmware marks them as "EEH incapable". So another

	 * two-step is needed to find the pci bus..

	 */

	if (!frozen_bus)

		frozen_bus = pcibios_find_pci_bus(frozen_dn->parent);



	frozen_bus = eeh_pe_bus_get(pe);

	if (!frozen_bus) {

		printk(KERN_ERR "EEH: Cannot find PCI bus "

		        "for location=%s dn=%s\n",

		        location, frozen_dn->full_name);

		return NULL;

		pr_err("%s: Cannot find PCI bus for PHB#%d-PE#%x\n",

			__func__, pe->phb->global_number, pe->addr);

		return;

	}



	frozen_edev = of_node_to_eeh_dev(frozen_dn);

	frozen_edev->freeze_count++;

	pci_str = eeh_pci_name(eeh_dev_to_pci_dev(event->edev));

	drv_str = eeh_pcid_name(eeh_dev_to_pci_dev(event->edev));



	if (frozen_edev->freeze_count > EEH_MAX_ALLOWED_FREEZES)

	pe->freeze_count++;

	if (pe->freeze_count > EEH_MAX_ALLOWED_FREEZES)

		goto excess_failures;



	printk(KERN_WARNING

	   "EEH: This PCI device has failed %d times in the last hour:\n",

		frozen_edev->freeze_count);



	if (frozen_edev->pdev) {

		bus_pci_str = pci_name(frozen_edev->pdev);

		bus_drv_str = eeh_pcid_name(frozen_edev->pdev);

		printk(KERN_WARNING

			"EEH: Bus location=%s driver=%s pci addr=%s\n",

			location, bus_drv_str, bus_pci_str);

	}



	printk(KERN_WARNING

		"EEH: Device location=%s driver=%s pci addr=%s\n",

		location, drv_str, pci_str);

	pr_warning("EEH: This PCI device has failed %d times in the last hour\n",

		pe->freeze_count);



	/* Walk the various device drivers attached to this slot through

	 * a reset sequence, giving each an opportunity to do what it needs
@@ -419,12 +382,12 @@ struct eeh_dev *handle_eeh_events(struct eeh_event *event) 
	 * status ... if any child can't handle the reset, then the entire

	 * slot is dlpar removed and added.

	 */

	pci_walk_bus(frozen_bus, eeh_report_error, &result);

	eeh_pe_dev_traverse(pe, eeh_report_error, &result);



	/* Get the current PCI slot state. This can take a long time,

	 * sometimes over 3 seconds for certain systems.

	 */

	rc = eeh_ops->wait_state(eeh_dev_to_of_node(frozen_edev), MAX_WAIT_FOR_RECOVERY*1000);

	rc = eeh_ops->wait_state(pe, MAX_WAIT_FOR_RECOVERY*1000);

	if (rc < 0 || rc == EEH_STATE_NOT_SUPPORT) {

		printk(KERN_WARNING "EEH: Permanent failure\n");

		goto hard_fail;
@@ -434,14 +397,14 @@ struct eeh_dev *handle_eeh_events(struct eeh_event *event) 
	 * don't post the error log until after all dev drivers

	 * have been informed.

	 */

	eeh_slot_error_detail(frozen_edev, EEH_LOG_TEMP);

	eeh_slot_error_detail(pe, EEH_LOG_TEMP);



	/* If all device drivers were EEH-unaware, then shut

	 * down all of the device drivers, and hope they

	 * go down willingly, without panicing the system.

	 */

	if (result == PCI_ERS_RESULT_NONE) {

		rc = eeh_reset_device(frozen_edev, frozen_bus);

		rc = eeh_reset_device(pe, frozen_bus);

		if (rc) {

			printk(KERN_WARNING "EEH: Unable to reset, rc=%d\n", rc);

			goto hard_fail;
@@ -450,7 +413,7 @@ struct eeh_dev *handle_eeh_events(struct eeh_event *event) 


	/* If all devices reported they can proceed, then re-enable MMIO */

	if (result == PCI_ERS_RESULT_CAN_RECOVER) {

		rc = eeh_pci_enable(frozen_edev, EEH_OPT_THAW_MMIO);

		rc = eeh_pci_enable(pe, EEH_OPT_THAW_MMIO);



		if (rc < 0)

			goto hard_fail;
@@ -458,13 +421,13 @@ struct eeh_dev *handle_eeh_events(struct eeh_event *event) 
			result = PCI_ERS_RESULT_NEED_RESET;

		} else {

			result = PCI_ERS_RESULT_NONE;

			pci_walk_bus(frozen_bus, eeh_report_mmio_enabled, &result);

			eeh_pe_dev_traverse(pe, eeh_report_mmio_enabled, &result);

		}

	}



	/* If all devices reported they can proceed, then re-enable DMA */

	if (result == PCI_ERS_RESULT_CAN_RECOVER) {

		rc = eeh_pci_enable(frozen_edev, EEH_OPT_THAW_DMA);

		rc = eeh_pci_enable(pe, EEH_OPT_THAW_DMA);



		if (rc < 0)

			goto hard_fail;
@@ -482,13 +445,13 @@ struct eeh_dev *handle_eeh_events(struct eeh_event *event) 


	/* If any device called out for a reset, then reset the slot */

	if (result == PCI_ERS_RESULT_NEED_RESET) {

		rc = eeh_reset_device(frozen_edev, NULL);

		rc = eeh_reset_device(pe, NULL);

		if (rc) {

			printk(KERN_WARNING "EEH: Cannot reset, rc=%d\n", rc);

			goto hard_fail;

		}

		result = PCI_ERS_RESULT_NONE;

		pci_walk_bus(frozen_bus, eeh_report_reset, &result);

		eeh_pe_dev_traverse(pe, eeh_report_reset, &result);

	}



	/* All devices should claim they have recovered by now. */
@@ -499,9 +462,9 @@ struct eeh_dev *handle_eeh_events(struct eeh_event *event) 
	}



	/* Tell all device drivers that they can resume operations */

	pci_walk_bus(frozen_bus, eeh_report_resume, NULL);

	eeh_pe_dev_traverse(pe, eeh_report_resume, NULL);



	return frozen_edev;

	return;

	

excess_failures:

	/*
@@ -509,30 +472,26 @@ struct eeh_dev *handle_eeh_events(struct eeh_event *event) 
	 * are due to poorly seated PCI cards. Only 10% or so are

	 * due to actual, failed cards.

	 */

	printk(KERN_ERR

	   "EEH: PCI device at location=%s driver=%s pci addr=%s\n"

		"has failed %d times in the last hour "

		"and has been permanently disabled.\n"

		"Please try reseating this device or replacing it.\n",

		location, drv_str, pci_str, frozen_edev->freeze_count);

	pr_err("EEH: PHB#%d-PE#%x has failed %d times in the\n"

	       "last hour and has been permanently disabled.\n"

	       "Please try reseating or replacing it.\n",

		pe->phb->global_number, pe->addr,

		pe->freeze_count);

	goto perm_error;



hard_fail:

	printk(KERN_ERR

	   "EEH: Unable to recover from failure of PCI device "

	   "at location=%s driver=%s pci addr=%s\n"

	   "Please try reseating this device or replacing it.\n",

		location, drv_str, pci_str);

	pr_err("EEH: Unable to recover from failure from PHB#%d-PE#%x.\n"

	       "Please try reseating or replacing it\n",

		pe->phb->global_number, pe->addr);



perm_error:

	eeh_slot_error_detail(frozen_edev, EEH_LOG_PERM);

	eeh_slot_error_detail(pe, EEH_LOG_PERM);



	/* Notify all devices that they're about to go down. */

	pci_walk_bus(frozen_bus, eeh_report_failure, NULL);

	eeh_pe_dev_traverse(pe, eeh_report_failure, NULL);



	/* Shut down the device drivers for good. */

	if (frozen_bus)

		pcibios_remove_pci_devices(frozen_bus);



	return NULL;

}

arch/powerpc/platforms/pseries/eeh_event.c

+1 −1
Original line number Diff line number Diff line
@@ -82,7 +82,7 @@ static int eeh_event_handler(void * dummy) 
		pe->phb->global_number, pe->addr);



	set_current_state(TASK_INTERRUPTIBLE);	/* Don't add to load average */

	handle_eeh_events(event);

	eeh_handle_event(pe);

	eeh_pe_state_clear(pe, EEH_PE_RECOVERING);



	kfree(event);

arch/powerpc/platforms/pseries/eeh_pe.c

+28 −0
Original line number Diff line number Diff line
@@ -561,3 +561,31 @@ void eeh_pe_restore_bars(struct eeh_pe *pe) 
{

	eeh_pe_dev_traverse(pe, eeh_restore_one_device_bars, NULL);

}



/**

 * eeh_pe_bus_get - Retrieve PCI bus according to the given PE

 * @pe: EEH PE

 *

 * Retrieve the PCI bus according to the given PE. Basically,

 * there're 3 types of PEs: PHB/Bus/Device. For PHB PE, the

 * primary PCI bus will be retrieved. The parent bus will be

 * returned for BUS PE. However, we don't have associated PCI

 * bus for DEVICE PE.

 */

struct pci_bus *eeh_pe_bus_get(struct eeh_pe *pe)

{

	struct pci_bus *bus = NULL;

	struct eeh_dev *edev;

	struct pci_dev *pdev;



	if (pe->type == EEH_PE_PHB) {

		bus = pe->phb->bus;

	} else if (pe->type == EEH_PE_BUS) {

		edev = list_first_entry(&pe->edevs, struct eeh_dev, list);

		pdev = eeh_dev_to_pci_dev(edev);

		if (pdev)

			bus = pdev->bus;

	}



	return bus;

}