Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit bb0361a5 authored by Sathish Ambley's avatar Sathish Ambley
Browse files

msm: ADSPRPC: Enable ADSP remote heap 



Enable dynamically growing and shrinking of ADSP heap
by allocating memory from Apps and locking the memory
to prevent Apps from being able to access the memory
while its in use by ADSP.

Change-Id: I7a05751c69b569ce3f1712c4bb1c941ab47c8174
Acked-by: default avatarBharath Kumar <bkumar@qti.qualcomm.com>
Signed-off-by: default avatarSathish Ambley <sathishambley@codeaurora.org>
parent 50e0173f

Documentation/devicetree/bindings/qdsp/msm-fastrpc.txt

+26 −5
Original line number Diff line number Diff line
@@ -60,14 +60,35 @@ Example: 
IOMMU Domain:



Required properties:

- compatible : Must be "qcom,iommu-domains"



Required subnode:

- adsp_shared_domain : IOMMU domain specifies the context bank and its VA range



Required subnode properties:

- label : Label describing the context bank

- qcom,iommu-contexts : phandle that describe the context bank handle

- qcom,virtual-addr-pool : Virtual address range that the context bank

                           will be using



Example:

	qcom,iommu-domains {

		compatible = "qcom,iommu-domains";

		adsp_shared_domain: qcom,iommu-domain6 {

			label = "adsp_shared";

			qcom,iommu-contexts = <&adsp_shared>;

			qcom,virtual-addr-pool = <0x80000000 0x7FFFFFFF>;

		};

	};



Remote Heap:



Required properties:

- compatible : Must be "qcom,msm-adsprpc-mem-region"

- memory-region : CMA region which is owned by this device



Example:

       qcom,adsprpc-mem {

               compatible = "qcom,msm-adsprpc-mem-region";

               memory-region = <&adsp_mem>;

       };

drivers/char/adsprpc.c

+291 −95
Original line number Diff line number Diff line
@@ -41,9 +41,15 @@ 
#include <linux/sort.h>

#include <linux/msm_dma_iommu_mapping.h>

#include <asm/dma-iommu.h>

#include <soc/qcom/scm.h>

#include "adsprpc_compat.h"

#include "adsprpc_shared.h"



#define TZ_PIL_PROTECT_MEM_SUBSYS_ID 0x0C

#define TZ_PIL_CLEAR_PROTECT_MEM_SUBSYS_ID 0x0D

#define TZ_PIL_AUTH_QDSP6_PROC 1

#define ADSP_MMAP_HEAP_ADDR 4



#define RPC_TIMEOUT	(5 * HZ)

#define BALIGN		128

#define NUM_CHANNELS	3		/*1 compute 1 cpz 1 mdsp*/
@@ -155,16 +161,19 @@ struct fastrpc_apps { 
	struct class *class;

	struct mutex smd_mutex;

	struct smq_phy_page range;

	struct hlist_head maps;

	dev_t dev_no;

	int compat;

	struct hlist_head drivers;

	spinlock_t hlock;

	struct ion_client *client;

	struct device *adsp_mem_device;

};



struct fastrpc_mmap {

	struct hlist_node hn;

	struct fastrpc_file *fl;

	struct fastrpc_apps *apps;

	int fd;

	uint32_t flags;

	struct dma_buf *buf;
@@ -246,21 +255,44 @@ static void fastrpc_buf_list_free(struct fastrpc_file *fl) 


static void fastrpc_mmap_add(struct fastrpc_mmap *map)

{

	if (map->flags == ADSP_MMAP_HEAP_ADDR) {

		struct fastrpc_apps *me = &gfa;



		spin_lock(&me->hlock);

		hlist_add_head(&map->hn, &me->maps);

		spin_unlock(&me->hlock);

	} else {

		struct fastrpc_file *fl = map->fl;



		spin_lock(&fl->hlock);

		hlist_add_head(&map->hn, &fl->maps);

		spin_unlock(&fl->hlock);

	}

}



static int fastrpc_mmap_find(struct fastrpc_file *fl, int fd, uintptr_t va,

			     ssize_t len, struct fastrpc_mmap **ppmap)

			ssize_t len, int mflags, struct fastrpc_mmap **ppmap)

{

	struct fastrpc_apps *me = &gfa;

	struct fastrpc_mmap *match = 0, *map;

	struct hlist_node *n;

	if (mflags == ADSP_MMAP_HEAP_ADDR) {

		spin_lock(&me->hlock);

		hlist_for_each_entry_safe(map, n, &me->maps, hn) {

			if (va >= map->va &&

				va + len <= map->va + map->len &&

				map->fd == fd) {

				map->refs++;

				match = map;

				break;

			}

		}

		spin_unlock(&me->hlock);

	} else {

		spin_lock(&fl->hlock);

		hlist_for_each_entry_safe(map, n, &fl->maps, hn) {

			if (va >= map->va &&

		    map->va + map->len <= va + len &&

				va + len <= map->va + map->len &&

				map->fd == fd) {

				map->refs++;

				match = map;
@@ -268,6 +300,7 @@ static int fastrpc_mmap_find(struct fastrpc_file *fl, int fd, uintptr_t va, 
			}

		}

		spin_unlock(&fl->hlock);

	}

	if (match) {

		*ppmap = match;

		return 0;
@@ -275,11 +308,51 @@ static int fastrpc_mmap_find(struct fastrpc_file *fl, int fd, uintptr_t va, 
	return -ENOTTY;

}



static int dma_alloc_memory(phys_addr_t *region_start, ssize_t size)

{

	struct fastrpc_apps *me = &gfa;

	void *vaddr = 0;

	DEFINE_DMA_ATTRS(attrs);



	if (me->adsp_mem_device == NULL) {

		pr_err("device adsprpc-mem is not initialized\n");

		return -ENODEV;

	}

	dma_set_attr(DMA_ATTR_SKIP_ZEROING, &attrs);

	dma_set_attr(DMA_ATTR_NO_KERNEL_MAPPING, &attrs);

	vaddr = dma_alloc_attrs(me->adsp_mem_device, size,

						region_start, GFP_KERNEL,

						&attrs);

	if (!vaddr) {

		pr_err("ADSPRPC: Failed to allocate %x remote heap memory\n",

						(unsigned int)size);

		return -ENOMEM;

	}

	return 0;

}



static int fastrpc_mmap_remove(struct fastrpc_file *fl, uintptr_t va,

			       ssize_t len, struct fastrpc_mmap **ppmap)

{

	struct fastrpc_mmap *match = 0, *map;

	struct hlist_node *n;

	struct fastrpc_apps *me = &gfa;



	spin_lock(&me->hlock);

	hlist_for_each_entry_safe(map, n, &me->maps, hn) {

		if (map->raddr == va &&

			map->raddr + map->len == va + len &&

			map->refs == 1) {

			match = map;

			hlist_del_init(&map->hn);

			break;

		}

	}

	spin_unlock(&me->hlock);

	if (match) {

		*ppmap = match;

		return 0;

	}

	spin_lock(&fl->hlock);

	hlist_for_each_entry_safe(map, n, &fl->maps, hn) {

		if (map->raddr == va &&
@@ -300,24 +373,44 @@ static int fastrpc_mmap_remove(struct fastrpc_file *fl, uintptr_t va, 


static void fastrpc_mmap_free(struct fastrpc_mmap *map)

{

	struct fastrpc_apps *me = &gfa;

	struct fastrpc_file *fl;

	if (!map)

		return;

	if (map->flags == ADSP_MMAP_HEAP_ADDR) {

		spin_lock(&me->hlock);

		map->refs--;

		if (!map->refs)

			hlist_del_init(&map->hn);

		spin_unlock(&me->hlock);

	} else {

		fl = map->fl;



		spin_lock(&fl->hlock);

		map->refs--;

		if (!map->refs)

			hlist_del_init(&map->hn);

		spin_unlock(&fl->hlock);

	}

	if (map->refs > 0)

		return;

	if (map->flags == ADSP_MMAP_HEAP_ADDR) {

		DEFINE_DMA_ATTRS(attrs);



	if (map->refs)

		if (me->adsp_mem_device != NULL) {

			pr_err("failed to free remote heap allocation\n");

			return;

		}

		dma_set_attr(DMA_ATTR_SKIP_ZEROING, &attrs);

		dma_set_attr(DMA_ATTR_NO_KERNEL_MAPPING, &attrs);

		dma_free_attrs(me->adsp_mem_device, map->size,

				&(map->va), map->phys,	&attrs);

	} else {

		if (!IS_ERR_OR_NULL(map->handle))

			ion_free(fl->apps->client, map->handle);

		if (map->size || map->phys)

			msm_dma_unmap_sg(fl->sctx->smmu.dev, map->table->sgl,

				map->table->nents, DMA_BIDIRECTIONAL, map->buf);

					map->table->nents, DMA_BIDIRECTIONAL,

					map->buf);

		if (!IS_ERR_OR_NULL(map->table))

			dma_buf_unmap_attachment(map->attach, map->table,

					DMA_BIDIRECTIONAL);
@@ -325,24 +418,39 @@ static void fastrpc_mmap_free(struct fastrpc_mmap *map) 
			dma_buf_detach(map->buf, map->attach);

		if (!IS_ERR_OR_NULL(map->buf))

			dma_buf_put(map->buf);

	}

	kfree(map);

}



static int fastrpc_mmap_create(struct fastrpc_file *fl, int fd, uintptr_t va,

			       ssize_t len, struct fastrpc_mmap **ppmap)

			ssize_t len, int mflags, struct fastrpc_mmap **ppmap)

{

	struct fastrpc_apps *me = &gfa;

	struct fastrpc_session_ctx *sess = fl->sctx;

	struct fastrpc_mmap *map = 0;

	struct dma_attrs attrs;

	phys_addr_t region_start = 0;

	unsigned long flags;

	int err = 0;

	if (!fastrpc_mmap_find(fl, fd, va, len, ppmap))



	if (!fastrpc_mmap_find(fl, fd, va, len, mflags, ppmap))

		return 0;

	VERIFY(err, map = kzalloc(sizeof(*map), GFP_KERNEL));

	map = kzalloc(sizeof(*map), GFP_KERNEL);

	VERIFY(err, !IS_ERR_OR_NULL(map));

	if (err)

		goto bail;

	map->flags = mflags;

	map->refs = 1;

	INIT_HLIST_NODE(&map->hn);

	map->fl = fl;

	map->fd = fd;

	if (mflags == ADSP_MMAP_HEAP_ADDR) {

		map->apps = me;

		map->fl = 0;

		VERIFY(err, !dma_alloc_memory(&region_start, len));

		map->phys = (uintptr_t)region_start;

		map->size = len;

	} else {

		VERIFY(err, !IS_ERR_OR_NULL(map->buf = dma_buf_get(fd)));

		if (err)

			goto bail;
@@ -357,7 +465,8 @@ static int fastrpc_mmap_create(struct fastrpc_file *fl, int fd, uintptr_t va, 
			goto bail;

		init_dma_attrs(&attrs);

		dma_set_attr(DMA_ATTR_EXEC_MAPPING, &attrs);

	VERIFY(err, map->table->nents == msm_dma_map_sg_attrs(sess->smmu.dev,

		VERIFY(err, map->table->nents ==

					msm_dma_map_sg_attrs(sess->smmu.dev,

					map->table->sgl, map->table->nents,

					DMA_BIDIRECTIONAL, map->buf, &attrs));

		if (err)
@@ -366,8 +475,8 @@ static int fastrpc_mmap_create(struct fastrpc_file *fl, int fd, uintptr_t va, 
				ion_import_dma_buf(fl->apps->client, fd)));

		if (err)

			goto bail;

	VERIFY(err, !ion_handle_get_flags(fl->apps->client, map->handle,

				&flags));

		VERIFY(err, !ion_handle_get_flags(fl->apps->client,

							map->handle, &flags));

		if (err)

			goto bail;

		map->uncached = !ION_IS_CACHED(flags);
@@ -377,14 +486,11 @@ static int fastrpc_mmap_create(struct fastrpc_file *fl, int fd, uintptr_t va, 
			map->phys += ((dma_addr_t)sess->smmu.cb << 32);

#endif

		map->size = sg_dma_len(map->table->sgl);

	map->refs = 1;

	}

	map->va = va;

	map->len = len;

	INIT_HLIST_NODE(&map->hn);



	spin_lock(&fl->hlock);

	hlist_add_head(&map->hn, &fl->maps);

	spin_unlock(&fl->hlock);

	fastrpc_mmap_add(map);

	*ppmap = map;



bail:
@@ -417,6 +523,7 @@ static int fastrpc_buf_alloc(struct fastrpc_file *fl, ssize_t size, 
		*obuf = fr;

		return 0;

	}

	buf = 0;

	VERIFY(err, buf = kzalloc(sizeof(*buf), GFP_KERNEL));

	if (err)

		goto bail;
@@ -427,8 +534,7 @@ static int fastrpc_buf_alloc(struct fastrpc_file *fl, ssize_t size, 
	buf->size = size;

	buf->virt = dma_alloc_coherent(fl->sctx->smmu.dev, buf->size,

				       &buf->phys, GFP_KERNEL);

	VERIFY(err, !IS_ERR_OR_NULL(buf->virt));

	if (err) {

	if (IS_ERR_OR_NULL(buf->virt)) {

		/* free cache and retry */

		fastrpc_buf_list_free(fl);

		buf->virt = dma_alloc_coherent(fl->sctx->smmu.dev, buf->size,
@@ -555,8 +661,7 @@ static int context_alloc(struct fastrpc_file *fl, uint32_t kernel, 
	struct fastrpc_ctx_lst *clst = &fl->clst;

	struct fastrpc_ioctl_invoke *invoke = &invokefd->inv;



	bufs = REMOTE_SCALARS_INBUFS(invoke->sc) +

			REMOTE_SCALARS_OUTBUFS(invoke->sc);

	bufs = REMOTE_SCALARS_LENGTH(invoke->sc);

	size = bufs * sizeof(*ctx->lpra) + bufs * sizeof(*ctx->maps) +

		sizeof(*ctx->fds) * (bufs) +

		sizeof(*ctx->overs) * (bufs) +
@@ -734,6 +839,7 @@ static int get_args(uint32_t kernel, struct smq_invoke_ctx *ctx) 
	ssize_t rlen = 0, copylen = 0, metalen = 0, size;

	int i, inh, oix;

	int err = 0;

	int mflags = 0;



	/* calculate size of the metadata */

	rpra = 0;
@@ -746,7 +852,7 @@ static int get_args(uint32_t kernel, struct smq_invoke_ctx *ctx) 
		ssize_t len = lpra[i].buf.len;

		if (ctx->fds[i])

			fastrpc_mmap_create(ctx->fl, ctx->fds[i], buf, len,

					    &ctx->maps[i]);

					    mflags, &ctx->maps[i]);

		ipage += 1;

	}

	metalen = copylen = (ssize_t)&ipage[0];
@@ -851,7 +957,6 @@ static int get_args(uint32_t kernel, struct smq_invoke_ctx *ctx) 
	for (oix = 0; oix < inbufs + outbufs; ++oix) {

		int i = ctx->overps[oix]->raix;

		struct fastrpc_mmap *map = ctx->maps[i];



		if (map && map->uncached)

			continue;

		if (rpra[i].buf.len && ctx->overps[oix]->mstart)
@@ -938,7 +1043,6 @@ static void inv_args(struct smq_invoke_ctx *ctx) 
	outbufs = REMOTE_SCALARS_OUTBUFS(sc);

	for (i = inbufs; i < inbufs + outbufs; ++i) {

		struct fastrpc_mmap *map = ctx->maps[i];



		if (map && map->uncached)

			continue;

		if (!rpra[i].buf.len)
@@ -1121,6 +1225,7 @@ static int fastrpc_init_process(struct fastrpc_file *fl, 
	} else if (init->flags == FASTRPC_INIT_CREATE) {

		remote_arg_t ra[4];

		int fds[4], i, len = 0;

		int mflags = 0;

		struct scatterlist *sg;

		struct {

			int pgid;
@@ -1132,12 +1237,12 @@ static int fastrpc_init_process(struct fastrpc_file *fl, 
		inbuf.namelen = strlen(current->comm);

		inbuf.filelen = init->filelen;

		VERIFY(err, !fastrpc_mmap_create(fl, init->filefd, init->file,

						 init->filelen, &file));

					init->filelen, mflags, &file));

		if (err)

			goto bail;

		inbuf.pageslen = 1;

		VERIFY(err, !fastrpc_mmap_create(fl, init->memfd, init->mem,

						 init->memlen, &mem));

					init->memlen, mflags, &mem));

		if (err)

			goto bail;

		for_each_sg(mem->table->sgl, sg, mem->table->nents, i) {
@@ -1258,10 +1363,55 @@ static int fastrpc_mmap_on_dsp(struct fastrpc_file *fl, uint32_t flags, 
	map->raddr = (uintptr_t)routargs.vaddrout;

	if (err)

		goto bail;

	if (flags == ADSP_MMAP_HEAP_ADDR) {

		struct scm_desc desc = {0};



		desc.args[0] = TZ_PIL_AUTH_QDSP6_PROC;

		desc.args[1] = map->phys;

		desc.args[2] = map->size;

		desc.arginfo = SCM_ARGS(3);

		err = scm_call2(SCM_SIP_FNID(SCM_SVC_PIL,

			TZ_PIL_PROTECT_MEM_SUBSYS_ID), &desc);

	}



bail:

	return err;

}



static int fastrpc_munmap_on_dsp_rh(struct fastrpc_file *fl,

				 struct fastrpc_mmap *map)

{

	struct fastrpc_ioctl_invoke_fd ioctl;

	struct scm_desc desc = {0};

	remote_arg_t ra[1];

	int err = 0;

	struct {

		uint8_t skey;

	} routargs;



	ra[0].buf.pv = (void *)&routargs;

	ra[0].buf.len = sizeof(routargs);



	ioctl.inv.handle = 1;

	ioctl.inv.sc = REMOTE_SCALARS_MAKE(7, 0, 1);

	ioctl.inv.pra = ra;

	ioctl.fds = 0;



	VERIFY(err, 0 == (err = fastrpc_internal_invoke(fl,

			FASTRPC_MODE_PARALLEL, 1, &ioctl)));

	if (err)

		goto bail;

	desc.args[0] = TZ_PIL_AUTH_QDSP6_PROC;

	desc.args[1] = map->phys;

	desc.args[2] = map->size;

	desc.args[3] = routargs.skey;

	desc.arginfo = SCM_ARGS(4);

	err = scm_call2(SCM_SIP_FNID(SCM_SVC_PIL,

		TZ_PIL_CLEAR_PROTECT_MEM_SUBSYS_ID), &desc);



bail:

	return err;

}



static int fastrpc_munmap_on_dsp(struct fastrpc_file *fl,

				 struct fastrpc_mmap *map)
@@ -1274,6 +1424,11 @@ static int fastrpc_munmap_on_dsp(struct fastrpc_file *fl, 
		uintptr_t vaddrout;

		ssize_t size;

	} inargs;

	if (map->flags == ADSP_MMAP_HEAP_ADDR) {

		VERIFY(err, !fastrpc_munmap_on_dsp_rh(fl, map));

		if (err)

			goto bail;

	}



	inargs.pid = current->tgid;

	inargs.size = map->size;
@@ -1290,6 +1445,34 @@ static int fastrpc_munmap_on_dsp(struct fastrpc_file *fl, 
	ioctl.fds = 0;

	VERIFY(err, 0 == (err = fastrpc_internal_invoke(fl,

		FASTRPC_MODE_PARALLEL, 1, &ioctl)));

bail:

	return err;

}



static int fastrpc_mmap_remove_ssr(struct fastrpc_file *fl)

{

	struct fastrpc_mmap *match = 0, *map = NULL;

	struct hlist_node *n = NULL;

	int err = 0;

	struct fastrpc_apps *me = &gfa;



	spin_lock(&me->hlock);

	hlist_for_each_entry_safe(map, n, &me->maps, hn) {

			match = map;

			hlist_del_init(&map->hn);

			break;

	}

	spin_unlock(&me->hlock);



	if (match) {

		VERIFY(err, !fastrpc_munmap_on_dsp_rh(fl, match));

		if (err)

			goto bail;

		fastrpc_mmap_free(match);

	}

bail:

	if (err && match)

		fastrpc_mmap_add(match);

	return err;

}
@@ -1323,11 +1506,11 @@ static int fastrpc_internal_mmap(struct fastrpc_file *fl, 
	struct fastrpc_mmap *map = 0;

	int err = 0;

	if (!fastrpc_mmap_find(fl, ud->fd, (uintptr_t)ud->vaddrin, ud->size,

			       &map))

			       ud->flags, &map))

		return 0;



	VERIFY(err, !fastrpc_mmap_create(fl, ud->fd, (uintptr_t)ud->vaddrin,

					 ud->size, &map));

					 ud->size, ud->flags, &map));

	if (err)

		goto bail;

	VERIFY(err, 0 == fastrpc_mmap_on_dsp(fl, ud->flags, map));
@@ -1414,11 +1597,14 @@ static int fastrpc_device_release(struct inode *inode, struct file *file) 
	struct fastrpc_apps *me = &gfa;

	struct fastrpc_file *fl = (struct fastrpc_file *)file->private_data;

	int session, cid;



	if (fl) {

		cid = fl->cid;

		session = fl->sctx - &me->channel[cid].session[0];

	fastrpc_file_free((struct fastrpc_file *)file->private_data);

		fastrpc_file_free(fl);

		file->private_data = 0;

		fastrpc_session_free(&me->channel[cid], session);

	}

	return 0;

}
@@ -1442,13 +1628,13 @@ static int fastrpc_device_open(struct inode *inode, struct file *filp) 
	INIT_HLIST_HEAD(&fl->maps);

	INIT_HLIST_HEAD(&fl->bufs);

	INIT_HLIST_NODE(&fl->hn);

	fl->tgid = current->tgid;

	fl->apps = me;

	VERIFY(err, !fastrpc_session_alloc(&me->channel[cid], &session));

	if (err)

		goto bail;

	fl->sctx = &me->channel[cid].session[session];

	fl->cid = cid;

	fl->tgid = current->tgid;

	fl->apps = me;



	fl->ssrcount = me->channel[cid].ssrcount;

	if ((kref_get_unless_zero(&me->channel[cid].kref) == 0) ||
@@ -1467,6 +1653,8 @@ static int fastrpc_device_open(struct inode *inode, struct file *filp) 
		kref_init(&me->channel[cid].kref);

		pr_info("'opened /dev/%s c %d %d'\n", gcinfo[cid].name,

						MAJOR(me->dev_no), cid);

		if (fastrpc_mmap_remove_ssr(fl))

			pr_err("ADSPRPC: SSR: Failed to unmap remote heap\n");

	}

	spin_lock(&me->hlock);

	hlist_add_head(&fl->hn, &me->drivers);
@@ -1596,6 +1784,7 @@ static struct of_device_id fastrpc_match_table[] = { 
	{ .compatible = "qcom,msm-fastrpc-adsp", },

	{ .compatible = "qcom,msm-fastrpc-compute-cb", },

	{ .compatible = "qcom,msm-fastrpc-legacy-compute-cb", },

	{ .compatible = "qcom,msm-adsprpc-mem-region", },

	{}

};
@@ -1735,6 +1924,7 @@ bail: 
static int fastrpc_probe(struct platform_device *pdev)

{

	int err = 0;

	struct fastrpc_apps *me = &gfa;

	struct device *dev = &pdev->dev;



	if (of_device_is_compatible(dev->of_node,
@@ -1745,6 +1935,12 @@ static int fastrpc_probe(struct platform_device *pdev) 
					"qcom,msm-fastrpc-legacy-compute-cb"))

		return fastrpc_cb_legacy_probe(dev);



	if (of_device_is_compatible(dev->of_node,

					"qcom,msm-adsprpc-mem-region")) {

		me->adsp_mem_device = dev;

		return 0;

	}



	VERIFY(err, !of_platform_populate(pdev->dev.of_node,

					  fastrpc_match_table,

					  NULL, &pdev->dev));
@@ -1797,6 +1993,7 @@ static int __init fastrpc_device_init(void) 
	memset(me, 0, sizeof(*me));



	fastrpc_init(me);

	me->adsp_mem_device = NULL;

	VERIFY(err, 0 == platform_driver_register(&fastrpc_driver));

	if (err)

		goto register_bail;
@@ -1836,7 +2033,6 @@ static int __init fastrpc_device_init(void) 
	if (err)

		goto device_create_bail;

	return 0;



device_create_bail:

	for (i = 0; i < NUM_CHANNELS; i++) {

		if (IS_ERR_OR_NULL(me->channel[i].dev))
@@ -1851,8 +2047,8 @@ class_create_bail: 
cdev_init_bail:

	unregister_chrdev_region(me->dev_no, NUM_CHANNELS);

alloc_chrdev_bail:

	fastrpc_deinit();

register_bail:

	fastrpc_deinit();

	return err;

}