Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 52815b75 authored by Joerg Roedel's avatar Joerg Roedel
Browse files

iommu/amd: Add support for IOMMUv2 domain mode



This patch adds support for protection domains that
implement two-level paging for devices.

Signed-off-by: default avatarJoerg Roedel <joerg.roedel@amd.com>
parent 132bd68f
Loading
Loading
Loading
Loading
+3 −1
Original line number Diff line number Diff line
@@ -34,7 +34,9 @@ config AMD_IOMMU
	bool "AMD IOMMU support"
	select SWIOTLB
	select PCI_MSI
	select PCI_IOV
	select PCI_ATS
	select PCI_PRI
	select PCI_PASID
	select IOMMU_API
	depends on X86_64 && PCI && ACPI
	---help---
+140 −4
Original line number Diff line number Diff line
@@ -63,6 +63,7 @@ static struct protection_domain *pt_domain;
static struct iommu_ops amd_iommu_ops;

static ATOMIC_NOTIFIER_HEAD(ppr_notifier);
int amd_iommu_max_glx_val = -1;

/*
 * general struct to manage commands send to an IOMMU
@@ -1598,6 +1599,11 @@ static void free_pagetable(struct protection_domain *domain)
	domain->pt_root = NULL;
}

static void free_gcr3_table(struct protection_domain *domain)
{
	free_page((unsigned long)domain->gcr3_tbl);
}

/*
 * Free a domain, only used if something went wrong in the
 * allocation path and we need to free an already allocated page table
@@ -1699,6 +1705,32 @@ static void set_dte_entry(u16 devid, struct protection_domain *domain, bool ats)
	if (ats)
		flags |= DTE_FLAG_IOTLB;

	if (domain->flags & PD_IOMMUV2_MASK) {
		u64 gcr3 = __pa(domain->gcr3_tbl);
		u64 glx  = domain->glx;
		u64 tmp;

		pte_root |= DTE_FLAG_GV;
		pte_root |= (glx & DTE_GLX_MASK) << DTE_GLX_SHIFT;

		/* First mask out possible old values for GCR3 table */
		tmp = DTE_GCR3_VAL_B(~0ULL) << DTE_GCR3_SHIFT_B;
		flags    &= ~tmp;

		tmp = DTE_GCR3_VAL_C(~0ULL) << DTE_GCR3_SHIFT_C;
		flags    &= ~tmp;

		/* Encode GCR3 table into DTE */
		tmp = DTE_GCR3_VAL_A(gcr3) << DTE_GCR3_SHIFT_A;
		pte_root |= tmp;

		tmp = DTE_GCR3_VAL_B(gcr3) << DTE_GCR3_SHIFT_B;
		flags    |= tmp;

		tmp = DTE_GCR3_VAL_C(gcr3) << DTE_GCR3_SHIFT_C;
		flags    |= tmp;
	}

	flags &= ~(0xffffUL);
	flags |= domain->id;

@@ -1803,6 +1835,46 @@ static int __attach_device(struct iommu_dev_data *dev_data,
	return ret;
}


static void pdev_iommuv2_disable(struct pci_dev *pdev)
{
	pci_disable_ats(pdev);
	pci_disable_pri(pdev);
	pci_disable_pasid(pdev);
}

static int pdev_iommuv2_enable(struct pci_dev *pdev)
{
	int ret;

	/* Only allow access to user-accessible pages */
	ret = pci_enable_pasid(pdev, 0);
	if (ret)
		goto out_err;

	/* First reset the PRI state of the device */
	ret = pci_reset_pri(pdev);
	if (ret)
		goto out_err;

	/* FIXME: Hardcode number of outstanding requests for now */
	ret = pci_enable_pri(pdev, 32);
	if (ret)
		goto out_err;

	ret = pci_enable_ats(pdev, PAGE_SHIFT);
	if (ret)
		goto out_err;

	return 0;

out_err:
	pci_disable_pri(pdev);
	pci_disable_pasid(pdev);

	return ret;
}

/*
 * If a device is not yet associated with a domain, this function does
 * assigns it visible for the hardware
@@ -1817,7 +1889,17 @@ static int attach_device(struct device *dev,

	dev_data = get_dev_data(dev);

	if (amd_iommu_iotlb_sup && pci_enable_ats(pdev, PAGE_SHIFT) == 0) {
	if (domain->flags & PD_IOMMUV2_MASK) {
		if (!dev_data->iommu_v2 || !dev_data->passthrough)
			return -EINVAL;

		if (pdev_iommuv2_enable(pdev) != 0)
			return -EINVAL;

		dev_data->ats.enabled = true;
		dev_data->ats.qdep    = pci_ats_queue_depth(pdev);
	} else if (amd_iommu_iotlb_sup &&
		   pci_enable_ats(pdev, PAGE_SHIFT) == 0) {
		dev_data->ats.enabled = true;
		dev_data->ats.qdep    = pci_ats_queue_depth(pdev);
	}
@@ -1877,21 +1959,25 @@ static void __detach_device(struct iommu_dev_data *dev_data)
 */
static void detach_device(struct device *dev)
{
	struct protection_domain *domain;
	struct iommu_dev_data *dev_data;
	unsigned long flags;

	dev_data = get_dev_data(dev);
	domain   = dev_data->domain;

	/* lock device table */
	write_lock_irqsave(&amd_iommu_devtable_lock, flags);
	__detach_device(dev_data);
	write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);

	if (dev_data->ats.enabled) {
	if (domain->flags & PD_IOMMUV2_MASK)
		pdev_iommuv2_disable(to_pci_dev(dev));
	else if (dev_data->ats.enabled)
		pci_disable_ats(to_pci_dev(dev));

	dev_data->ats.enabled = false;
}
}

/*
 * Find out the protection domain structure for a given PCI device. This
@@ -2788,6 +2874,9 @@ static void amd_iommu_domain_destroy(struct iommu_domain *dom)
	if (domain->mode != PAGE_MODE_NONE)
		free_pagetable(domain);

	if (domain->flags & PD_IOMMUV2_MASK)
		free_gcr3_table(domain);

	protection_domain_free(domain);

	dom->priv = NULL;
@@ -3010,3 +3099,50 @@ void amd_iommu_domain_direct_map(struct iommu_domain *dom)
	spin_unlock_irqrestore(&domain->lock, flags);
}
EXPORT_SYMBOL(amd_iommu_domain_direct_map);

int amd_iommu_domain_enable_v2(struct iommu_domain *dom, int pasids)
{
	struct protection_domain *domain = dom->priv;
	unsigned long flags;
	int levels, ret;

	if (pasids <= 0 || pasids > (PASID_MASK + 1))
		return -EINVAL;

	/* Number of GCR3 table levels required */
	for (levels = 0; (pasids - 1) & ~0x1ff; pasids >>= 9)
		levels += 1;

	if (levels > amd_iommu_max_glx_val)
		return -EINVAL;

	spin_lock_irqsave(&domain->lock, flags);

	/*
	 * Save us all sanity checks whether devices already in the
	 * domain support IOMMUv2. Just force that the domain has no
	 * devices attached when it is switched into IOMMUv2 mode.
	 */
	ret = -EBUSY;
	if (domain->dev_cnt > 0 || domain->flags & PD_IOMMUV2_MASK)
		goto out;

	ret = -ENOMEM;
	domain->gcr3_tbl = (void *)get_zeroed_page(GFP_ATOMIC);
	if (domain->gcr3_tbl == NULL)
		goto out;

	domain->glx      = levels;
	domain->flags   |= PD_IOMMUV2_MASK;
	domain->updated  = true;

	update_domain(domain);

	ret = 0;

out:
	spin_unlock_irqrestore(&domain->lock, flags);

	return ret;
}
EXPORT_SYMBOL(amd_iommu_domain_enable_v2);
+9 −0
Original line number Diff line number Diff line
@@ -755,6 +755,7 @@ static void __init init_iommu_from_pci(struct amd_iommu *iommu)
	iommu->features = ((u64)high << 32) | low;

	if (iommu_feature(iommu, FEATURE_GT)) {
		int glxval;
		u32 pasids;
		u64 shift;

@@ -763,6 +764,14 @@ static void __init init_iommu_from_pci(struct amd_iommu *iommu)
		pasids  = (1 << shift);

		amd_iommu_max_pasids = min(amd_iommu_max_pasids, pasids);

		glxval   = iommu->features & FEATURE_GLXVAL_MASK;
		glxval >>= FEATURE_GLXVAL_SHIFT;

		if (amd_iommu_max_glx_val == -1)
			amd_iommu_max_glx_val = glxval;
		else
			amd_iommu_max_glx_val = min(amd_iommu_max_glx_val, glxval);
	}

	if (iommu_feature(iommu, FEATURE_GT) &&
+1 −0
Original line number Diff line number Diff line
@@ -39,6 +39,7 @@ extern bool amd_iommu_v2_supported(void);
extern int amd_iommu_register_ppr_notifier(struct notifier_block *nb);
extern int amd_iommu_unregister_ppr_notifier(struct notifier_block *nb);
extern void amd_iommu_domain_direct_map(struct iommu_domain *dom);
extern int amd_iommu_domain_enable_v2(struct iommu_domain *dom, int pasids);

#ifndef CONFIG_AMD_IOMMU_STATS

+27 −0
Original line number Diff line number Diff line
@@ -93,6 +93,11 @@
#define FEATURE_PASID_SHIFT	32
#define FEATURE_PASID_MASK	(0x1fULL << FEATURE_PASID_SHIFT)

#define FEATURE_GLXVAL_SHIFT	14
#define FEATURE_GLXVAL_MASK	(0x03ULL << FEATURE_GLXVAL_SHIFT)

#define PASID_MASK		0x000fffff

/* MMIO status bits */
#define MMIO_STATUS_COM_WAIT_INT_MASK	(1 << 2)
#define MMIO_STATUS_PPR_INT_MASK	(1 << 6)
@@ -257,6 +262,22 @@
#define IOMMU_PTE_IW (1ULL << 62)

#define DTE_FLAG_IOTLB	(0x01UL << 32)
#define DTE_FLAG_GV	(0x01ULL << 55)
#define DTE_GLX_SHIFT	(56)
#define DTE_GLX_MASK	(3)

#define DTE_GCR3_VAL_A(x)	(((x) >> 12) & 0x00007ULL)
#define DTE_GCR3_VAL_B(x)	(((x) >> 15) & 0x0ffffULL)
#define DTE_GCR3_VAL_C(x)	(((x) >> 31) & 0xfffffULL)

#define DTE_GCR3_INDEX_A	0
#define DTE_GCR3_INDEX_B	1
#define DTE_GCR3_INDEX_C	1

#define DTE_GCR3_SHIFT_A	58
#define DTE_GCR3_SHIFT_B	16
#define DTE_GCR3_SHIFT_C	43


#define IOMMU_PAGE_MASK (((1ULL << 52) - 1) & ~0xfffULL)
#define IOMMU_PTE_PRESENT(pte) ((pte) & IOMMU_PTE_P)
@@ -283,6 +304,7 @@
					      domain for an IOMMU */
#define PD_PASSTHROUGH_MASK	(1UL << 2) /* domain has no page
					      translation */
#define PD_IOMMUV2_MASK		(1UL << 3) /* domain has gcr3 table */

extern bool amd_iommu_dump;
#define DUMP_printk(format, arg...)					\
@@ -344,6 +366,8 @@ struct protection_domain {
	u16 id;			/* the domain id written to the device table */
	int mode;		/* paging mode (0-6 levels) */
	u64 *pt_root;		/* page table root pointer */
	int glx;		/* Number of levels for GCR3 table */
	u64 *gcr3_tbl;		/* Guest CR3 table */
	unsigned long flags;	/* flags to find out type of domain */
	bool updated;		/* complete domain flush required */
	unsigned dev_cnt;	/* devices assigned to this domain */
@@ -611,6 +635,9 @@ extern bool amd_iommu_v2_present;

extern bool amd_iommu_force_isolation;

/* Max levels of glxval supported */
extern int amd_iommu_max_glx_val;

/* takes bus and device/function and returns the device id
 * FIXME: should that be in generic PCI code? */
static inline u16 calc_devid(u8 bus, u8 devfn)