Merge "iommu: msm: Fix potential page table use after free" (e1dfe27e) · Commits · e / devices / android_kernel_sony_msm8994

drivers/iommu/msm_iommu-v1.c

+4 −0

Original line number	Diff line number	Diff line
		@@ -994,6 +994,8 @@ static size_t msm_iommu_unmap(struct iommu_domain *domain, unsigned long va,
		goto fail;

		ret = __flush_iotlb_va(domain, va);

		msm_iommu_pagetable_free_tables(&priv->pt, va, len);
		fail:
		mutex_unlock(&msm_iommu_lock);

		@@ -1039,6 +1041,8 @@ static int msm_iommu_unmap_range(struct iommu_domain *domain, unsigned int va,
		msm_iommu_pagetable_unmap_range(&priv->pt, va, len);

		__flush_iotlb(domain);

		msm_iommu_pagetable_free_tables(&priv->pt, va, len);
		mutex_unlock(&msm_iommu_lock);
		return 0;
		}

drivers/iommu/msm_iommu_pagetable.c

+68 −25

Original line number	Diff line number	Diff line
		@@ -100,7 +100,15 @@ int msm_iommu_pagetable_alloc(struct msm_iommu_pt *pt)
		if (!pt->fl_table)
		return -ENOMEM;

		pt->fl_table_shadow = (u32 *)__get_free_pages(GFP_KERNEL,
		get_order(SZ_16K));
		if (!pt->fl_table_shadow) {
		free_pages((unsigned long)pt->fl_table, get_order(SZ_16K));
		return -ENOMEM;
		}

		memset(pt->fl_table, 0, SZ_16K);
		memset(pt->fl_table_shadow, 0, SZ_16K);
		clean_pte(pt->fl_table, pt->fl_table + NUM_FL_PTE, pt->redirect);

		return 0;
		@@ -109,15 +117,45 @@ int msm_iommu_pagetable_alloc(struct msm_iommu_pt *pt)
		void msm_iommu_pagetable_free(struct msm_iommu_pt *pt)
		{
		u32 *fl_table;
		u32 *fl_table_shadow;
		int i;

		fl_table = pt->fl_table;
		fl_table_shadow = pt->fl_table_shadow;
		for (i = 0; i < NUM_FL_PTE; i++)
		if ((fl_table[i] & 0x03) == FL_TYPE_TABLE)
		free_page((unsigned long) __va(((fl_table[i]) &
		FL_BASE_MASK)));
		free_pages((unsigned long)fl_table, get_order(SZ_16K));
		pt->fl_table = 0;

		free_pages((unsigned long)fl_table_shadow, get_order(SZ_16K));
		pt->fl_table_shadow = 0;
		}

		void msm_iommu_pagetable_free_tables(struct msm_iommu_pt *pt, unsigned long va,
		size_t len)
		{
		/*
		* Adding 2 for worst case. We could be spanning 3 second level pages
		* if we unmapped just over 1MB.
		*/
		u32 n_entries = len / SZ_1M + 2;
		u32 fl_offset = FL_OFFSET(va);
		u32 i;

		for (i = 0; i < n_entries && fl_offset < NUM_FL_PTE; ++i) {
		u32 *fl_pte_shadow = pt->fl_table_shadow + fl_offset;
		void sl_table_va = __va(((fl_pte_shadow) & ~0x1FF));
		u32 sl_table = *fl_pte_shadow;

		if (sl_table && !(sl_table & 0x1FF)) {
		free_pages((unsigned long) sl_table_va,
		get_order(SZ_4K));
		*fl_pte_shadow = 0;
		}
		++fl_offset;
		}
		}

		static int __get_pgprot(int prot, int len)
		@@ -162,8 +200,8 @@ static int __get_pgprot(int prot, int len)
		return pgprot;
		}

		static u32 make_second_level(struct msm_iommu_pt pt,
		u32 *fl_pte)
		static u32 make_second_level(struct msm_iommu_pt pt, u32 *fl_pte,
		u32 *fl_pte_shadow)
		{
		u32 *sl;
		sl = (u32 *) __get_free_pages(GFP_KERNEL,
		@@ -178,6 +216,7 @@ static u32 make_second_level(struct msm_iommu_pt pt,

		*fl_pte = ((((int)__pa(sl)) & FL_BASE_MASK) \| \
		FL_TYPE_TABLE);
		fl_pte_shadow = fl_pte & ~0x1FF;

		clean_pte(fl_pte, fl_pte + 1, pt->redirect);
		fail:
		@@ -368,6 +407,7 @@ int msm_iommu_pagetable_map_range(struct msm_iommu_pt *pt, unsigned int va,
		unsigned int start_va = va;
		unsigned int offset = 0;
		u32 *fl_pte;
		u32 *fl_pte_shadow;
		u32 fl_offset;
		u32 *sl_table = NULL;
		u32 sl_offset, sl_start;
		@@ -388,6 +428,7 @@ int msm_iommu_pagetable_map_range(struct msm_iommu_pt *pt, unsigned int va,

		fl_offset = FL_OFFSET(va); /* Upper 12 bits */
		fl_pte = pt->fl_table + fl_offset; /* int pointers, 4 bytes */
		fl_pte_shadow = pt->fl_table_shadow + fl_offset;
		pa = get_phys_addr(sg);

		ret = check_range(pt->fl_table, va, len);
		@@ -415,12 +456,14 @@ int msm_iommu_pagetable_map_range(struct msm_iommu_pt *pt, unsigned int va,
		goto fail;
		clean_pte(fl_pte, fl_pte + 16, pt->redirect);
		fl_pte += 16;
		fl_pte_shadow += 16;
		} else if (chunk_size == SZ_1M) {
		ret = fl_1m(fl_pte, pa, pgprot1m);
		if (ret)
		goto fail;
		clean_pte(fl_pte, fl_pte + 1, pt->redirect);
		fl_pte++;
		fl_pte_shadow++;
		}

		offset += chunk_size;
		@@ -437,7 +480,7 @@ int msm_iommu_pagetable_map_range(struct msm_iommu_pt *pt, unsigned int va,
		}
		/* for 4K or 64K, make sure there is a second level table */
		if (*fl_pte == 0) {
		if (!make_second_level(pt, fl_pte)) {
		if (!make_second_level(pt, fl_pte, fl_pte_shadow)) {
		ret = -ENOMEM;
		goto fail;
		}
		@@ -471,13 +514,16 @@ int msm_iommu_pagetable_map_range(struct msm_iommu_pt *pt, unsigned int va,
		if (chunk_size == SZ_4K) {
		sl_4k(&sl_table[sl_offset], pa, pgprot4k);
		sl_offset++;
		/* Increment map count */
		(*fl_pte_shadow)++;
		} else {
		BUG_ON(sl_offset + 16 > NUM_SL_PTE);
		sl_64k(&sl_table[sl_offset], pa, pgprot64k);
		sl_offset += 16;
		/* Increment map count */
		*fl_pte_shadow += 16;
		}


		offset += chunk_size;
		chunk_offset += chunk_size;
		va += chunk_size;
		@@ -493,6 +539,7 @@ int msm_iommu_pagetable_map_range(struct msm_iommu_pt *pt, unsigned int va,
		clean_pte(sl_table + sl_start, sl_table + sl_offset,
		pt->redirect);
		fl_pte++;
		fl_pte_shadow++;
		sl_offset = 0;
		}

		@@ -508,64 +555,60 @@ void msm_iommu_pagetable_unmap_range(struct msm_iommu_pt *pt, unsigned int va,
		{
		unsigned int offset = 0;
		u32 *fl_pte;
		u32 *fl_pte_shadow;
		u32 fl_offset;
		u32 *sl_table;
		u32 sl_start, sl_end;
		int used, i;
		int used;

		BUG_ON(len & (SZ_4K - 1));

		fl_offset = FL_OFFSET(va); /* Upper 12 bits */
		fl_pte = pt->fl_table + fl_offset; /* int pointers, 4 bytes */
		fl_pte_shadow = pt->fl_table_shadow + fl_offset;

		while (offset < len) {
		if (*fl_pte & FL_TYPE_TABLE) {
		unsigned int n_entries;

		sl_start = SL_OFFSET(va);
		sl_table = __va(((*fl_pte) & FL_BASE_MASK));
		sl_end = ((len - offset) / SZ_4K) + sl_start;

		if (sl_end > NUM_SL_PTE)
		sl_end = NUM_SL_PTE;
		n_entries = sl_end - sl_start;

		memset(sl_table + sl_start, 0, (sl_end - sl_start) * 4);
		memset(sl_table + sl_start, 0, n_entries * 4);
		clean_pte(sl_table + sl_start, sl_table + sl_end,
		pt->redirect);

		offset += (sl_end - sl_start) * SZ_4K;
		va += (sl_end - sl_start) * SZ_4K;
		offset += n_entries * SZ_4K;
		va += n_entries * SZ_4K;

		/* Unmap and free the 2nd level table if all mappings
		* in it were removed. This saves memory, but the table
		* will need to be re-allocated the next time someone
		* tries to map these VAs.
		*/
		used = 0;
		BUG_ON((*fl_pte_shadow & 0x1FF) < n_entries);

		/* Decrement map count */
		*fl_pte_shadow -= n_entries;
		used = *fl_pte_shadow & 0x1FF;

		/* If we just unmapped the whole table, don't bother
		* seeing if there are still used entries left.
		*/
		if (sl_end - sl_start != NUM_SL_PTE)
		for (i = 0; i < NUM_SL_PTE; i++)
		if (sl_table[i]) {
		used = 1;
		break;
		}
		if (!used) {
		free_page((unsigned long)sl_table);
		*fl_pte = 0;

		clean_pte(fl_pte, fl_pte + 1, pt->redirect);
		}

		sl_start = 0;
		} else {
		*fl_pte = 0;
		*fl_pte_shadow = 0;

		clean_pte(fl_pte, fl_pte + 1, pt->redirect);
		va += SZ_1M;
		offset += SZ_1M;
		sl_start = 0;
		}
		fl_pte++;
		fl_pte_shadow++;
		}
		}

drivers/iommu/msm_iommu_pagetable.h

+3 −1

Original line number	Diff line number	Diff line
		/* Copyright (c) 2012-2013, The Linux Foundation. All rights reserved.
		/* Copyright (c) 2012-2014, The Linux Foundation. All rights reserved.
		*
		* This program is free software; you can redistribute it and/or modify
		* it under the terms of the GNU General Public License version 2 and
		@@ -28,4 +28,6 @@ void msm_iommu_pagetable_unmap_range(struct msm_iommu_pt *pt, unsigned int va,
		unsigned int len);
		phys_addr_t msm_iommu_iova_to_phys_soft(struct iommu_domain *domain,
		phys_addr_t va);
		void msm_iommu_pagetable_free_tables(struct msm_iommu_pt *pt, unsigned long va,
		size_t len);
		#endif

drivers/iommu/msm_iommu_pagetable_lpae.c

+101 −56

Original line number	Diff line number	Diff line
		@@ -107,6 +107,11 @@ s32 msm_iommu_pagetable_alloc(struct msm_iommu_pt *pt)
		fl_table_phys = ALIGN(fl_table_phys, FL_ALIGN);
		pt->fl_table = phys_to_virt(fl_table_phys);

		pt->sl_table_shadow = kzalloc(sizeof(u64 ) NUM_FL_PTE, GFP_KERNEL);
		if (!pt->sl_table_shadow) {
		kfree(pt->unaligned_fl_table);
		return -ENOMEM;
		}
		clean_pte(pt->fl_table, pt->fl_table + NUM_FL_PTE, pt->redirect);
		return 0;
		}
		@@ -121,12 +126,51 @@ void msm_iommu_pagetable_free(struct msm_iommu_pt *pt)
		u64 p = fl_table[i] & FLSL_BASE_MASK;
		free_page((u32)phys_to_virt(p));
		}
		if ((pt->sl_table_shadow[i]))
		free_page((u32)pt->sl_table_shadow[i]);
		}
		kfree(pt->unaligned_fl_table);

		pt->unaligned_fl_table = 0;
		pt->fl_table = 0;

		kfree(pt->sl_table_shadow);
		}

		void msm_iommu_pagetable_free_tables(struct msm_iommu_pt *pt, unsigned long va,
		size_t len)
		{
		/*
		* Adding 2 for worst case. We could be spanning 3 second level pages
		* if we unmapped just over 2MB.
		*/
		u32 n_entries = len / SZ_2M + 2;
		u32 fl_offset = FL_OFFSET(va);
		u32 sl_offset = SL_OFFSET(va);
		u32 i;

		for (i = 0; i < n_entries && fl_offset < NUM_FL_PTE; ++i) {
		void *tl_table_va;
		u32 entry;
		u64 *sl_pte_shadow;

		sl_pte_shadow = pt->sl_table_shadow[fl_offset] + sl_offset;
		entry = *sl_pte_shadow;
		tl_table_va = __va(((*sl_pte_shadow) & ~0xFFF));

		if (entry && !(entry & 0xFFF)) {
		free_page((unsigned long)tl_table_va);
		*sl_pte_shadow = 0;
		}
		++sl_offset;
		if (sl_offset >= NUM_TL_PTE) {
		sl_offset = 0;
		++fl_offset;
		}
		}
		}


		#ifdef CONFIG_ARM_LPAE
		/*
		* If LPAE is enabled in the ARM processor then just use the same
		@@ -182,25 +226,36 @@ static inline void __get_attr(s32 prot, u64 upper_attr, u64 lower_attr)
		*lower_attr \|= (prot & IOMMU_WRITE) ? TL_AP_RW : TL_AP_RO;
		}

		static inline u64 make_second_level_tbl(s32 redirect, u64 fl_pte)
		static inline u64 make_second_level_tbl(struct msm_iommu_pt pt, u32 offset)
		{
		u64 sl = (u64 ) __get_free_page(GFP_KERNEL);
		u64 *fl_pte = pt->fl_table + offset;

		if (!sl) {
		pr_err("Could not allocate second level table\n");
		goto fail;
		}

		pt->sl_table_shadow[offset] = (u64 *) __get_free_page(GFP_KERNEL);
		if (!pt->sl_table_shadow[offset]) {
		free_page((u32) sl);
		pr_err("Could not allocate second level shadow table\n");
		goto fail;
		}

		memset(sl, 0, SZ_4K);
		clean_pte(sl, sl + NUM_SL_PTE, redirect);
		memset(pt->sl_table_shadow[offset], 0, SZ_4K);
		clean_pte(sl, sl + NUM_SL_PTE, pt->redirect);

		/* Leave APTable bits 0 to let next level decide access permissinons */
		*fl_pte = (((phys_addr_t)__pa(sl)) & FLSL_BASE_MASK) \| FLSL_TYPE_TABLE;
		clean_pte(fl_pte, fl_pte + 1, redirect);
		clean_pte(fl_pte, fl_pte + 1, pt->redirect);
		fail:
		return sl;
		}

		static inline u64 make_third_level_tbl(s32 redirect, u64 sl_pte)
		static inline u64 make_third_level_tbl(s32 redirect, u64 sl_pte,
		u64 *sl_pte_shadow)
		{
		u64 tl = (u64 ) __get_free_page(GFP_KERNEL);

		@@ -213,7 +268,7 @@ static inline u64 make_third_level_tbl(s32 redirect, u64 sl_pte)

		/* Leave APTable bits 0 to let next level decide access permissions */
		*sl_pte = (((phys_addr_t)__pa(tl)) & FLSL_BASE_MASK) \| FLSL_TYPE_TABLE;

		sl_pte_shadow = sl_pte & ~0xFFF;
		clean_pte(sl_pte, sl_pte + 1, redirect);
		fail:
		return tl;
		@@ -332,17 +387,20 @@ static inline s32 common_error_check(size_t len, u64 const *fl_table)
		return ret;
		}

		static inline s32 handle_1st_lvl(u64 *fl_pte, phys_addr_t pa, u64 upper_attr,
		u64 lower_attr, size_t len, s32 redirect)
		static inline s32 handle_1st_lvl(struct msm_iommu_pt *pt, u32 offset,
		phys_addr_t pa, size_t len, u64 upper_attr,
		u64 lower_attr)
		{
		s32 ret = 0;
		u64 *fl_pte = pt->fl_table + offset;

		if (len == SZ_1G) {
		ret = fl_1G_map(fl_pte, pa, upper_attr, lower_attr, redirect);
		ret = fl_1G_map(fl_pte, pa, upper_attr, lower_attr,
		pt->redirect);
		} else {
		/* Need second level page table */
		if (*fl_pte == 0) {
		if (make_second_level_tbl(redirect, fl_pte) == NULL)
		if (make_second_level_tbl(pt, offset) == NULL)
		ret = -ENOMEM;
		}
		if (!ret) {
		@@ -353,18 +411,20 @@ static inline s32 handle_1st_lvl(u64 *fl_pte, phys_addr_t pa, u64 upper_attr,
		return ret;
		}

		static inline s32 handle_3rd_lvl(u64 *sl_pte, u32 va, phys_addr_t pa,
		u64 upper_attr, u64 lower_attr, size_t len,
		s32 redirect)
		static inline s32 handle_3rd_lvl(u64 sl_pte, u64 sl_pte_shadow, u32 va,
		phys_addr_t pa, u64 upper_attr,
		u64 lower_attr, size_t len, s32 redirect)
		{
		u64 *tl_table;
		u64 *tl_pte;
		u32 tl_offset;
		s32 ret = 0;
		u32 n_entries;

		/* Need a 3rd level table */
		if (*sl_pte == 0) {
		if (make_third_level_tbl(redirect, sl_pte) == NULL) {
		if (make_third_level_tbl(redirect, sl_pte, sl_pte_shadow)
		== NULL) {
		ret = -ENOMEM;
		goto fail;
		}
		@@ -379,10 +439,17 @@ static inline s32 handle_3rd_lvl(u64 *sl_pte, u32 va, phys_addr_t pa,
		tl_offset = TL_OFFSET(va);
		tl_pte = tl_table + tl_offset;

		if (len == SZ_64K)
		if (len == SZ_64K) {
		ret = tl_64k_map(tl_pte, pa, upper_attr, lower_attr, redirect);
		else
		n_entries = 16;
		} else {
		ret = tl_4k_map(tl_pte, pa, upper_attr, lower_attr, redirect);
		n_entries = 1;
		}

		/* Increment map count */
		if (!ret)
		*sl_pte_shadow += n_entries;

		fail:
		return ret;
		@@ -408,27 +475,6 @@ fail:
		return ret;
		}

		static u32 free_table(u64 prev_level_pte, u64 table, u32 table_len,
		s32 redirect, u32 check)
		{
		u32 i;
		u32 used = 0;

		if (check) {
		for (i = 0; i < table_len; ++i)
		if (table[i]) {
		used = 1;
		break;
		}
		}
		if (!used) {
		free_page((u32)table);
		*prev_level_pte = 0;
		clean_pte(prev_level_pte, prev_level_pte + 1, redirect);
		}
		return !used;
		}

		static void fl_1G_unmap(u64 *fl_pte, s32 redirect)
		{
		*fl_pte = 0;
		@@ -479,6 +525,7 @@ s32 msm_iommu_pagetable_map_range(struct msm_iommu_pt *pt, u32 va,
		u32 offset = 0;
		u64 *fl_pte;
		u64 *sl_pte;
		u64 *sl_pte_shadow;
		u32 fl_offset;
		u32 sl_offset;
		u64 *sl_table = NULL;
		@@ -519,22 +566,24 @@ s32 msm_iommu_pagetable_map_range(struct msm_iommu_pt *pt, u32 va,

		trace_iommu_map_range(va, pa, sg->length, chunk_size);

		ret = handle_1st_lvl(fl_pte, pa, up_at, lo_at,
		chunk_size, redirect);
		ret = handle_1st_lvl(pt, fl_offset, pa, chunk_size,
		up_at, lo_at);
		if (ret)
		goto fail;

		sl_table = FOLLOW_TO_NEXT_TABLE(fl_pte);
		sl_offset = SL_OFFSET(va);
		sl_pte = sl_table + sl_offset;
		sl_pte_shadow = pt->sl_table_shadow[fl_offset] + sl_offset;

		if (chunk_size == SZ_32M)
		ret = sl_32m_map(sl_pte, pa, up_at, lo_at, redirect);
		else if (chunk_size == SZ_2M)
		ret = sl_2m_map(sl_pte, pa, up_at, lo_at, redirect);
		else if (chunk_size == SZ_64K \|\| chunk_size == SZ_4K)
		ret = handle_3rd_lvl(sl_pte, va, pa, up_at, lo_at,
		chunk_size, redirect);
		ret = handle_3rd_lvl(sl_pte, sl_pte_shadow, va, pa,
		up_at, lo_at, chunk_size,
		redirect);
		if (ret)
		goto fail;

		@@ -578,7 +627,6 @@ static void __msm_iommu_pagetable_unmap_range(struct msm_iommu_pt *pt, u32 va,

		while (offset < len) {
		u32 entries;
		u32 check;
		u32 left_to_unmap = len - offset;
		u32 type;

		@@ -608,13 +656,11 @@ static void __msm_iommu_pagetable_unmap_range(struct msm_iommu_pt *pt, u32 va,

		clean_pte(sl_pte, sl_pte + 1, redirect);

		free_table(fl_pte, sl_table, NUM_SL_PTE,
		redirect, 1);

		offset += SZ_2M;
		va += SZ_2M;
		} else if (type == FLSL_TYPE_TABLE) {
		u32 tbl_freed;
		u64 *sl_pte_shadow =
		pt->sl_table_shadow[fl_offset] + sl_offset;

		tl_start = TL_OFFSET(va);
		tl_table = FOLLOW_TO_NEXT_TABLE(sl_pte);
		@@ -631,17 +677,16 @@ static void __msm_iommu_pagetable_unmap_range(struct msm_iommu_pt *pt, u32 va,
		clean_pte(tl_table + tl_start,
		tl_table + tl_end, redirect);

		/* If we just unmapped the whole table, don't
		* bother seeing if there are still used
		* entries left.
		*/
		check = entries != NUM_TL_PTE;
		BUG_ON((*sl_pte_shadow & 0xFFF) < entries);

		/* Decrement map count */
		*sl_pte_shadow -= entries;

		tbl_freed = free_table(sl_pte, tl_table,
		NUM_TL_PTE, redirect, check);
		if (tbl_freed)
		free_table(fl_pte, sl_table, NUM_SL_PTE,
		redirect, 1);
		if (!(*sl_pte_shadow & 0xFFF)) {
		*sl_pte = 0;
		clean_pte(sl_pte, sl_pte + 1,
		pt->redirect);
		}

		offset += entries * SZ_4K;
		va += entries * SZ_4K;

drivers/iommu/msm_iommu_priv.h

+14 −0

Original line number	Diff line number	Diff line
		@@ -21,10 +21,23 @@
		* unaligned_fl_table: Original address of memory for the page table.
		* fl_table is manually aligned (as per spec) but we need the original address
		* to free the table.
		* fl_table_shadow: This is "copy" of the fl_table with some differences.
		* It stores the same information as fl_table except that instead of storing
		* second level page table address + page table entry descriptor bits it
		* stores the second level page table address and the number of used second
		* level page tables entries. This is used to check whether we need to free
		* the second level page table which allows us to also free the second level
		* page table after doing a TLB invalidate which should catch bugs with
		* clients trying to unmap an address that is being used.
		* fl_table_shadow will use the lower 9 bits for the use count and the upper
		* bits for the second level page table address.
		* sl_table_shadow uses the same concept as fl_table_shadow but for LPAE 2nd
		* level page tables.
		*/
		#ifdef CONFIG_IOMMU_LPAE
		struct msm_iommu_pt {
		u64 *fl_table;
		u64 **sl_table_shadow;
		int redirect;
		u64 *unaligned_fl_table;
		};
		@@ -32,6 +45,7 @@ struct msm_iommu_pt {
		struct msm_iommu_pt {
		u32 *fl_table;
		int redirect;
		u32 *fl_table_shadow;
		};
		#endif
		/**