Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 7ff7ffe7 authored by Jordan Crouse's avatar Jordan Crouse Committed by Gerrit - the friendly Code Review server
Browse files

msm: kgsl: Make global memory statistics atomic 



If you run aggressive memory operations long enough, eventually you
will notice that the KGSL memory statistics will explode up into the
3G range. This is a race condition on the statistics math which ends up
going negative in certain cases.  Turning the statistics variables into
atomics should solve the problem once and for all. ONCE AND FOR ALL.

Change-Id: Ic0dedbad61762d667550c22190c4f9d0b453829b
Signed-off-by: default avatarJordan Crouse <jcrouse@codeaurora.org>
parent d9a88e39

drivers/gpu/msm/kgsl.c

+17 −5
Original line number Diff line number Diff line
@@ -232,7 +232,8 @@ kgsl_mem_entry_destroy(struct kref *kref) 
	kgsl_mem_entry_detach_process(entry);



	if (memtype != KGSL_MEM_ENTRY_KERNEL)

		kgsl_driver.stats.mapped -= entry->memdesc.size;

		atomic_long_sub(entry->memdesc.size,

			&kgsl_driver.stats.mapped);



	/*

	 * Ion takes care of freeing the sg_table for us so
@@ -2214,8 +2215,8 @@ long kgsl_ioctl_gpuobj_import(struct kgsl_device_private *dev_priv, 


	param->id = entry->id;



	KGSL_STATS_ADD(entry->memdesc.size, kgsl_driver.stats.mapped,

		kgsl_driver.stats.mapped_max);

	KGSL_STATS_ADD(entry->memdesc.size, &kgsl_driver.stats.mapped,

		&kgsl_driver.stats.mapped_max);



	kgsl_process_add_stats(private,

		kgsl_memdesc_usermem_type(&entry->memdesc),
@@ -2483,8 +2484,8 @@ long kgsl_ioctl_map_user_mem(struct kgsl_device_private *dev_priv, 
	param->gpuaddr = (unsigned long)

		entry->memdesc.gpuaddr + (param->offset & PAGE_MASK);



	KGSL_STATS_ADD(param->len, kgsl_driver.stats.mapped,

		kgsl_driver.stats.mapped_max);

	KGSL_STATS_ADD(param->len, &kgsl_driver.stats.mapped,

		&kgsl_driver.stats.mapped_max);



	kgsl_process_add_stats(private,

			kgsl_memdesc_usermem_type(&entry->memdesc), param->len);
@@ -3565,6 +3566,17 @@ struct kgsl_driver kgsl_driver = { 
	 * 8064 and 8974 once the region to be flushed is > 16mb.

	 */

	.full_cache_threshold = SZ_16M,



	.stats.vmalloc = ATOMIC_LONG_INIT(0),

	.stats.vmalloc_max = ATOMIC_LONG_INIT(0),

	.stats.page_alloc = ATOMIC_LONG_INIT(0),

	.stats.page_alloc_max = ATOMIC_LONG_INIT(0),

	.stats.coherent = ATOMIC_LONG_INIT(0),

	.stats.coherent_max = ATOMIC_LONG_INIT(0),

	.stats.secure = ATOMIC_LONG_INIT(0),

	.stats.secure_max = ATOMIC_LONG_INIT(0),

	.stats.mapped = ATOMIC_LONG_INIT(0),

	.stats.mapped_max = ATOMIC_LONG_INIT(0),

};

EXPORT_SYMBOL(kgsl_driver);

drivers/gpu/msm/kgsl.h

+18 −12
Original line number Diff line number Diff line
@@ -44,8 +44,14 @@ 
   the statisic is greater then _max, set _max

*/



#define KGSL_STATS_ADD(_size, _stat, _max) \

	do { _stat += (_size); if (_stat > _max) _max = _stat; } while (0)

static inline void KGSL_STATS_ADD(uint64_t size, atomic_long_t *stat,

		atomic_long_t *max)

{

	uint64_t ret = atomic_long_add_return(size, stat);



	if (ret > atomic_long_read(max))

		atomic_long_set(max, ret);

}



#define KGSL_MAX_NUMIBS 100000
@@ -76,16 +82,16 @@ struct kgsl_driver { 
	struct mutex devlock;



	struct {

		uint64_t vmalloc;

		uint64_t vmalloc_max;

		uint64_t page_alloc;

		uint64_t page_alloc_max;

		uint64_t coherent;

		uint64_t coherent_max;

		uint64_t secure;

		uint64_t secure_max;

		uint64_t mapped;

		uint64_t mapped_max;

		atomic_long_t vmalloc;

		atomic_long_t vmalloc_max;

		atomic_long_t page_alloc;

		atomic_long_t page_alloc_max;

		atomic_long_t coherent;

		atomic_long_t coherent_max;

		atomic_long_t secure;

		atomic_long_t secure_max;

		atomic_long_t mapped;

		atomic_long_t mapped_max;

	} stats;

	unsigned int full_cache_threshold;

};

drivers/gpu/msm/kgsl_mmu.c

+28 −25
Original line number Diff line number Diff line
@@ -344,8 +344,11 @@ sysfs_show_entries(struct kobject *kobj, 


	pt = _get_pt_from_kobj(kobj);



	if (pt)

		ret += snprintf(buf, PAGE_SIZE, "%d\n", pt->stats.entries);

	if (pt) {

		unsigned int val = atomic_read(&pt->stats.entries);



		ret += snprintf(buf, PAGE_SIZE, "%d\n", val);

	}



	kgsl_put_pagetable(pt);

	return ret;
@@ -361,8 +364,11 @@ sysfs_show_mapped(struct kobject *kobj, 


	pt = _get_pt_from_kobj(kobj);



	if (pt)

		ret += snprintf(buf, PAGE_SIZE, "%llu\n", pt->stats.mapped);

	if (pt) {

		uint64_t val = atomic_long_read(&pt->stats.mapped);



		ret += snprintf(buf, PAGE_SIZE, "%llu\n", val);

	}



	kgsl_put_pagetable(pt);

	return ret;
@@ -378,8 +384,11 @@ sysfs_show_max_mapped(struct kobject *kobj, 


	pt = _get_pt_from_kobj(kobj);



	if (pt)

		ret += snprintf(buf, PAGE_SIZE, "%llu\n", pt->stats.max_mapped);

	if (pt) {

		uint64_t val = atomic_long_read(&pt->stats.max_mapped);



		ret += snprintf(buf, PAGE_SIZE, "%llu\n", val);

	}



	kgsl_put_pagetable(pt);

	return ret;
@@ -611,6 +620,10 @@ kgsl_mmu_createpagetableobject(struct kgsl_mmu *mmu, 
	pagetable->name = name;

	pagetable->fault_addr = 0xFFFFFFFF;



	atomic_set(&pagetable->stats.entries, 0);

	atomic_long_set(&pagetable->stats.mapped, 0);

	atomic_long_set(&pagetable->stats.max_mapped, 0);



	if (mmu->mmu_ops && mmu->mmu_ops->mmu_init_pt) {

		status = mmu->mmu_ops->mmu_init_pt(mmu, pagetable);

		if (status)
@@ -754,22 +767,16 @@ kgsl_mmu_map(struct kgsl_pagetable *pagetable, 
		size += kgsl_memdesc_guard_page_size(memdesc);



	ret = pagetable->pt_ops->mmu_map(pagetable, memdesc);

	spin_lock(&pagetable->lock);



	if (ret)

		goto done;

	if (ret == 0) {

		KGSL_STATS_ADD(size, &pagetable->stats.mapped,

			&pagetable->stats.max_mapped);



	KGSL_STATS_ADD(size, pagetable->stats.mapped,

		       pagetable->stats.max_mapped);

	pagetable->stats.entries++;

		atomic_inc(&pagetable->stats.entries);



	spin_unlock(&pagetable->lock);

		memdesc->priv |= KGSL_MEMDESC_MAPPED;

	}



	return 0;



done:

	spin_unlock(&pagetable->lock);

	return ret;

}

EXPORT_SYMBOL(kgsl_mmu_map);
@@ -835,8 +842,6 @@ kgsl_mmu_unmap(struct kgsl_pagetable *pagetable, 
	start_addr = memdesc->gpuaddr;

	end_addr = (memdesc->gpuaddr + size);



	if (KGSL_MMU_TYPE_IOMMU != kgsl_mmu_get_mmutype())

		spin_lock(&pagetable->lock);

	pagetable->pt_ops->mmu_unmap(pagetable, memdesc);



	/* If buffer is unmapped 0 fault addr */
@@ -844,15 +849,13 @@ kgsl_mmu_unmap(struct kgsl_pagetable *pagetable, 
		(pagetable->fault_addr < end_addr))

		pagetable->fault_addr = 0;



	if (KGSL_MMU_TYPE_IOMMU == kgsl_mmu_get_mmutype())

		spin_lock(&pagetable->lock);

	/* Remove the statistics */

	pagetable->stats.entries--;

	pagetable->stats.mapped -= size;

	atomic_dec(&pagetable->stats.entries);

	atomic_long_sub(size, &pagetable->stats.mapped);



	spin_unlock(&pagetable->lock);

	if (!kgsl_memdesc_is_global(memdesc))

		memdesc->priv &= ~KGSL_MEMDESC_MAPPED;



	return 0;

}

EXPORT_SYMBOL(kgsl_mmu_unmap);

drivers/gpu/msm/kgsl_mmu.h

+3 −3
Original line number Diff line number Diff line
@@ -43,9 +43,9 @@ struct kgsl_pagetable { 
	struct kobject *kobj;



	struct {

		unsigned int entries;

		uint64_t mapped;

		uint64_t max_mapped;

		atomic_t entries;

		atomic_long_t mapped;

		atomic_long_t max_mapped;

	} stats;

	const struct kgsl_mmu_pt_ops *pt_ops;

	unsigned int fault_addr;

drivers/gpu/msm/kgsl_sharedmem.c

+33 −26
Original line number Diff line number Diff line
@@ -215,25 +215,25 @@ static ssize_t kgsl_drv_memstat_show(struct device *dev, 
	uint64_t val = 0;



	if (!strcmp(attr->attr.name, "vmalloc"))

		val = kgsl_driver.stats.vmalloc;

		val = atomic_long_read(&kgsl_driver.stats.vmalloc);

	else if (!strcmp(attr->attr.name, "vmalloc_max"))

		val = kgsl_driver.stats.vmalloc_max;

		val = atomic_long_read(&kgsl_driver.stats.vmalloc_max);

	else if (!strcmp(attr->attr.name, "page_alloc"))

		val = kgsl_driver.stats.page_alloc;

		val = atomic_long_read(&kgsl_driver.stats.page_alloc);

	else if (!strcmp(attr->attr.name, "page_alloc_max"))

		val = kgsl_driver.stats.page_alloc_max;

		val = atomic_long_read(&kgsl_driver.stats.page_alloc_max);

	else if (!strcmp(attr->attr.name, "coherent"))

		val = kgsl_driver.stats.coherent;

		val = atomic_long_read(&kgsl_driver.stats.coherent);

	else if (!strcmp(attr->attr.name, "coherent_max"))

		val = kgsl_driver.stats.coherent_max;

		val = atomic_long_read(&kgsl_driver.stats.coherent_max);

	else if (!strcmp(attr->attr.name, "secure"))

		val = kgsl_driver.stats.secure;

		val = atomic_long_read(&kgsl_driver.stats.secure);

	else if (!strcmp(attr->attr.name, "secure_max"))

		val = kgsl_driver.stats.secure_max;

		val = atomic_long_read(&kgsl_driver.stats.secure_max);

	else if (!strcmp(attr->attr.name, "mapped"))

		val = kgsl_driver.stats.mapped;

		val = atomic_long_read(&kgsl_driver.stats.mapped);

	else if (!strcmp(attr->attr.name, "mapped_max"))

		val = kgsl_driver.stats.mapped_max;

		val = atomic_long_read(&kgsl_driver.stats.mapped_max);



	return snprintf(buf, PAGE_SIZE, "%llu\n", val);

}
@@ -399,7 +399,8 @@ static void kgsl_page_alloc_unmap_kernel(struct kgsl_memdesc *memdesc) 
	if (memdesc->hostptr_count)

		goto done;

	vunmap(memdesc->hostptr);

	kgsl_driver.stats.vmalloc -= (unsigned long) memdesc->size;



	atomic_long_sub(memdesc->size, &kgsl_driver.stats.vmalloc);

	memdesc->hostptr = NULL;

done:

	mutex_unlock(&kernel_map_global_lock);
@@ -410,6 +411,8 @@ static void kgsl_page_alloc_free(struct kgsl_memdesc *memdesc) 
	unsigned int i = 0;

	struct scatterlist *sg;



	atomic_long_sub(memdesc->size, &kgsl_driver.stats.page_alloc);



	kgsl_page_alloc_unmap_kernel(memdesc);

	/* we certainly do not expect the hostptr to still be mapped */

	BUG_ON(memdesc->hostptr);
@@ -428,9 +431,10 @@ static void kgsl_page_alloc_free(struct kgsl_memdesc *memdesc) 
					memdesc->gpuaddr, memdesc->size, ret);

			BUG();

		}

		kgsl_driver.stats.secure -= memdesc->size;



		atomic_long_sub(memdesc->size, &kgsl_driver.stats.secure);

	} else {

		kgsl_driver.stats.page_alloc -= (size_t) memdesc->size;

		atomic_long_sub(memdesc->size, &kgsl_driver.stats.page_alloc);

	}



	for_each_sg(memdesc->sgt->sgl, sg, memdesc->sgt->nents, i) {
@@ -501,9 +505,9 @@ static int kgsl_page_alloc_map_kernel(struct kgsl_memdesc *memdesc) 
		memdesc->hostptr = vmap(pages, count,

					VM_IOREMAP, page_prot);

		if (memdesc->hostptr)

			KGSL_STATS_ADD((size_t) memdesc->size,

				kgsl_driver.stats.vmalloc,

				kgsl_driver.stats.vmalloc_max);

			KGSL_STATS_ADD(memdesc->size,

				&kgsl_driver.stats.vmalloc,

				&kgsl_driver.stats.vmalloc_max);

		else

			ret = -ENOMEM;

		kgsl_free(pages);
@@ -543,11 +547,14 @@ static void kgsl_cma_coherent_free(struct kgsl_memdesc *memdesc) 


	if (memdesc->hostptr) {

		if (memdesc->priv & KGSL_MEMDESC_SECURE) {

			kgsl_driver.stats.secure -= memdesc->size;

			atomic_long_sub(memdesc->size,

				&kgsl_driver.stats.secure);



			kgsl_cma_unlock_secure(memdesc);

			attrs = &memdesc->attrs;

		} else

			kgsl_driver.stats.coherent -= (size_t) memdesc->size;

			atomic_long_sub(memdesc->size,

				&kgsl_driver.stats.coherent);



		dma_free_attrs(memdesc->dev, (size_t) memdesc->size,

			memdesc->hostptr, memdesc->physaddr, attrs);
@@ -789,8 +796,8 @@ _kgsl_sharedmem_page_alloc(struct kgsl_memdesc *memdesc, 
		memdesc->priv |= KGSL_MEMDESC_TZ_LOCKED;



		/* Record statistics */

		KGSL_STATS_ADD(memdesc->size, kgsl_driver.stats.secure,

			kgsl_driver.stats.secure_max);

		KGSL_STATS_ADD(memdesc->size, &kgsl_driver.stats.secure,

			&kgsl_driver.stats.secure_max);



		/* Don't map and zero the locked secure buffer */

		goto done;
@@ -838,8 +845,8 @@ _kgsl_sharedmem_page_alloc(struct kgsl_memdesc *memdesc, 
		}

	}



	KGSL_STATS_ADD(memdesc->size, kgsl_driver.stats.page_alloc,

		kgsl_driver.stats.page_alloc_max);

	KGSL_STATS_ADD(memdesc->size, &kgsl_driver.stats.page_alloc,

		&kgsl_driver.stats.page_alloc_max);



done:

	if (ret) {
@@ -1084,8 +1091,8 @@ int kgsl_cma_alloc_coherent(struct kgsl_device *device, 


	/* Record statistics */



	KGSL_STATS_ADD(size, kgsl_driver.stats.coherent,

		       kgsl_driver.stats.coherent_max);

	KGSL_STATS_ADD(size, &kgsl_driver.stats.coherent,

		&kgsl_driver.stats.coherent_max);



err:

	if (result)
@@ -1207,8 +1214,8 @@ int kgsl_cma_alloc_secure(struct kgsl_device *device, 
	memdesc->priv |= KGSL_MEMDESC_TZ_LOCKED;



	/* Record statistics */

	KGSL_STATS_ADD(aligned, kgsl_driver.stats.secure,

	       kgsl_driver.stats.secure_max);

	KGSL_STATS_ADD(aligned, &kgsl_driver.stats.secure,

	       &kgsl_driver.stats.secure_max);

err:

	if (result)

		kgsl_sharedmem_free(memdesc);