Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 584608c1 authored by Rama Krishna Phani A's avatar Rama Krishna Phani A
Browse files

dmaengine: edma: add protection for variables to avoid corruption 



Changes include support for

i) Add protection for variables being used across functions to avoid
corruption.

ii) Updation of CB flag
EDMA hardware will start processing descriptors based on CB flag. During
high throughput conditions there can be a chance that CB flags get
updated even before memcpy of descriptor data element. Hence make sure
to set the CB flag post memcpy.

iii) Add support for dumping EDMA registers.

Change-Id: I97bba62b71536014ef8b3ceef2862654e98dc821
Signed-off-by: default avatarRama Krishna Phani A <rphani@codeaurora.org>
parent bab04d12

drivers/dma/qcom/pci-edma.c

+109 −10
Original line number Diff line number Diff line 
/* Copyright (c) 2019, The Linux Foundation. All rights reserved.

/* Copyright (c) 2019-2020, The Linux Foundation. All rights reserved.

 *

 * This program is free software; you can redistribute it and/or modify

 * it under the terms of the GNU General Public License version 2 and
@@ -32,6 +32,7 @@ 
#include <linux/sched_clock.h>

#include <linux/slab.h>

#include <linux/vmalloc.h>

#include <linux/msm_ep_pcie.h>

#include "../dmaengine.h"



/* global logging macros */
@@ -95,6 +96,8 @@ enum debug_log_lvl { 
#define EDMA_DRV_NAME "edma"

#define DEFAULT_KLOG_LVL (LOG_LVL_ERROR)



static struct edma_dev *e_dev_info;



#ifdef CONFIG_QCOM_PCI_EDMA_DEBUG

#define DEFAULT_IPC_LOG_LVL (LOG_LVL_VERBOSE)

#define IPC_LOG_PAGES (40)
@@ -172,6 +175,7 @@ enum debug_log_lvl { 
#define EDMA_HW_STATUS_MASK (BIT(6) | BIT(5))

#define EDMA_HW_STATUS_SHIFT (5)

#define EDMA_INT_ERR_MASK (0xff00)

#define EDMA_MAX_SIZE 0x2000



#define REQ_OF_DMA_ARGS (2) /* # of arguments required from client */
@@ -278,6 +282,7 @@ struct edmac_dev { 
	struct edma_dev *e_dev;

	u32 n_ev;

	u32 ch_id;

	spinlock_t edma_lock;

	enum edma_dir_ch dir;

	dma_addr_t tl_dma_rd_p; /* address of next DE to process */

	struct data_element *tl_wr_p; /* next available DE in TL */
@@ -365,13 +370,15 @@ static void edmac_process_tasklet(unsigned long data) 
	struct edmac_dev *ec_dev = (struct edmac_dev *)data;

	struct edma_dev *e_dev = ec_dev->e_dev;

	dma_addr_t llp_low, llp_high, llp;

	unsigned long flags;



	spin_lock_irqsave(&ec_dev->edma_lock, flags);

	EDMAC_VERB(ec_dev, EDMAV_NO_CH_ID, "enter\n");



	llp_low = readl_relaxed(ec_dev->llp_low_reg);

	llp_high = readl_relaxed(ec_dev->llp_high_reg);

	llp = (u64)(llp_high << 32) | llp_low;

	EDMAC_VERB(ec_dev, EDMAV_NO_CH_ID, "DMA_LLP: %pad\n", &llp);

	EDMAC_VERB(ec_dev, EDMAV_NO_CH_ID, "Start: DMA_LLP = %pad\n", &llp);



	while (ec_dev->tl_dma_rd_p != llp) {

		struct edma_desc *desc;
@@ -397,18 +404,30 @@ static void edmac_process_tasklet(unsigned long data) 
			&ec_dev->tl_dma_rd_p);



		desc = *ec_dev->dl_rd_p;

		ec_dev->tl_dma_rd_p += sizeof(struct data_element);

		ec_dev->tl_rd_p++;

		ec_dev->dl_rd_p++;

		if (desc) {

			/*

			 * Clients might queue descriptors in the call back

			 * context. Release spinlock to avoid deadlock scenarios

			 * as we use same lock duing descriptor queuing.

			 */

			spin_unlock_irqrestore(&ec_dev->edma_lock, flags);

			dmaengine_desc_get_callback_invoke(&desc->tx, NULL);



			/* Acquire spinlock again to continue edma operations */

			spin_lock_irqsave(&ec_dev->edma_lock, flags);

			kfree(desc);

		} else {

			EDMAC_VERB(ec_dev, EDMAV_NO_CH_ID,

				"edma desc is NULL\n");

		}



		ec_dev->tl_dma_rd_p += sizeof(struct data_element);

		ec_dev->tl_rd_p++;

		ec_dev->dl_rd_p++;

	}



	edma_set_clear(ec_dev->int_mask_reg, 0, BIT(ec_dev->ch_id));

	EDMAC_VERB(ec_dev, EDMAV_NO_CH_ID, "exit\n");

	spin_unlock_irqrestore(&ec_dev->edma_lock, flags);

}



static irqreturn_t handle_edma_irq(int irq, void *data)
@@ -653,10 +672,9 @@ static inline void edma_compose_data_element(struct edmav_dev *ev_dev, 
				unsigned long flags)

{

	EDMAC_VERB(ev_dev->ec_dev, ev_dev->ch_id,

		"dst_addr: %pad\tsrc_addr: %pad\n",

		&dst_addr, &src_addr);

		"size = %d, dst_addr: %pad\tsrc_addr: %pad\n",

		size, &dst_addr, &src_addr);



	de->ch_ctrl |= EDMA_CH_CONTROL1_CB;

	if (flags & DMA_PREP_INTERRUPT)

		de->ch_ctrl |= EDMA_CH_CONTROL1_LIE;

	de->size = size;
@@ -688,7 +706,9 @@ struct dma_async_tx_descriptor *edma_prep_dma_memcpy(struct dma_chan *chan, 
{

	struct edmav_dev *ev_dev = to_edmav_dev(chan);

	struct edma_desc *desc;

	unsigned long l_flags;



	spin_lock_irqsave(&ev_dev->ec_dev->edma_lock, l_flags);

	EDMAC_VERB(ev_dev->ec_dev, ev_dev->ch_id, "enter\n");



	desc = edma_alloc_descriptor(ev_dev);
@@ -701,10 +721,13 @@ struct dma_async_tx_descriptor *edma_prep_dma_memcpy(struct dma_chan *chan, 
	list_add_tail(&desc->node, &ev_dev->dl);



	EDMAC_VERB(ev_dev->ec_dev, ev_dev->ch_id, "exit\n");

	spin_unlock_irqrestore(&ev_dev->ec_dev->edma_lock, l_flags);



	return &desc->tx;

err:

	EDMAC_VERB(ev_dev->ec_dev, ev_dev->ch_id, "exit with error\n");

	EDMAC_VERB(ev_dev->ec_dev, ev_dev->ch_id,

		"edma alloc descriptor failed for channel:%d\n", ev_dev->ch_id);

	spin_unlock_irqrestore(&ev_dev->ec_dev->edma_lock, l_flags);

	return NULL;

}
@@ -715,6 +738,28 @@ static void edma_issue_descriptor(struct edmac_dev *ec_dev, 
	*ec_dev->dl_wr_p = desc;

	memcpy(ec_dev->tl_wr_p, desc->de, sizeof(*desc->de));



	/*

	 * Ensure Desc data element should be flushed to tl_wr_p

	 * before updating CB flag

	 */

	mb();



	EDMAC_VERB(ec_dev, EDMAV_NO_CH_ID,

			"size: %d, dst_addr: %pad\tsrc_addr: %pad\n",

			ec_dev->tl_wr_p->size, &ec_dev->tl_wr_p->dar,

			&ec_dev->tl_wr_p->sar);



	ec_dev->tl_wr_p->ch_ctrl |= EDMA_CH_CONTROL1_CB;



	/*

	 * Ensure that CB flag is properly flushed because

	 * HW starts processing the descriptor based on CB flag.

	 */

	mb();



	EDMAC_VERB(ec_dev, EDMAV_NO_CH_ID, "ch_ctrl = %d\n",

		ec_dev->tl_wr_p->ch_ctrl);



	ec_dev->dl_wr_p++;

	ec_dev->tl_wr_p++;

	ec_dev->n_de_avail--;
@@ -739,11 +784,14 @@ static void edma_issue_pending(struct dma_chan *chan) 
	struct edmac_dev *ec_dev = ev_dev->ec_dev;

	struct edma_desc *desc;

	enum edma_hw_state hw_state;

	unsigned long flags;



	spin_lock_irqsave(&ec_dev->edma_lock, flags);

	EDMAC_VERB(ec_dev, ev_dev->ch_id, "enter\n");



	if (unlikely(list_empty(&ev_dev->dl))) {

		EDMAC_VERB(ec_dev, ev_dev->ch_id, "No descriptor to issue\n");

		spin_unlock_irqrestore(&ec_dev->edma_lock, flags);

		return;

	}
@@ -755,12 +803,42 @@ static void edma_issue_pending(struct dma_chan *chan) 
	hw_state = edma_get_hw_state(ec_dev);

	if ((hw_state == EDMA_HW_STATE_STOPPED) ||

		(hw_state == EDMA_HW_STATE_INIT)) {

		/* Disable Engine and enable it back */

		writel_relaxed(false, ec_dev->engine_en_reg);

		writel_relaxed(true, ec_dev->engine_en_reg);

		EDMAC_VERB(ec_dev, ev_dev->ch_id,

			"Channel stopped: Disable Engine and enable back\n");

		/* Ensure that engine is restarted */

		mb();



		/*

		 * From spec, when channel is stopped,

		 * require to write llp reg to start edma transaction

		 */

		writel_relaxed(

			readl_relaxed(ec_dev->llp_low_reg),

			ec_dev->llp_low_reg);

		writel_relaxed(

			readl_relaxed(ec_dev->llp_high_reg),

			ec_dev->llp_high_reg);

		/* Ensure LLP registers are properly updated */

		mb();



		/*

		 * As Channel is stopped, to start edma transaction,

		 * ring channel doorbell

		 */

		EDMAC_VERB(ec_dev, EDMAV_NO_CH_ID, "ringing doorbell\n");

		writel_relaxed(ec_dev->ch_id, ec_dev->db_reg);

		ec_dev->hw_state = EDMA_HW_STATE_ACTIVE;

		/* Ensure DB register is properly updated */

		mb();

	} else {

		EDMAC_VERB(ec_dev, ev_dev->ch_id, "EDMA Channel is Active\n");

	}



	EDMAC_VERB(ec_dev, ev_dev->ch_id, "exit\n");

	spin_unlock_irqrestore(&ec_dev->edma_lock, flags);

}



static int edma_config(struct dma_chan *chan, struct dma_slave_config *config)
@@ -890,6 +968,7 @@ static void edma_init_channels(struct edma_dev *e_dev) 
		ec_dev->llp_low_reg = e_dev->base + llp_low_off;

		ec_dev->llp_high_reg = e_dev->base + llp_high_off;

		ec_dev->db_reg = e_dev->base + db_off;

		spin_lock_init(&ec_dev->edma_lock);

	}



	/* setup virtual channels */
@@ -927,6 +1006,25 @@ static void edma_init_dma_device(struct edma_dev *e_dev) 
	e_dev->dma_device.device_tx_status = dma_cookie_status;

}



void edma_dump(void)

{

	int i;



	for (i = 0; i < EDMA_MAX_SIZE; i += 32) {

		pr_err("EDMA Reg : 0x%04x %08x %08x %08x %08x %08x %08x %08x %08x\n",

			i,

			readl_relaxed(e_dev_info->base + i),

			readl_relaxed(e_dev_info->base + (i + 4)),

			readl_relaxed(e_dev_info->base + (i + 8)),

			readl_relaxed(e_dev_info->base + (i + 12)),

			readl_relaxed(e_dev_info->base + (i + 16)),

			readl_relaxed(e_dev_info->base + (i + 20)),

			readl_relaxed(e_dev_info->base + (i + 24)),

			readl_relaxed(e_dev_info->base + (i + 28)));

	}

}

EXPORT_SYMBOL(edma_dump);



/*

 * Initializes and enables eDMA driver and H/W block for PCIe controllers.

 * Only call this function if PCIe supports eDMA and has all its resources
@@ -1052,6 +1150,7 @@ int qcom_edma_init(struct device *dev) 
		return ret;



	edma_init_dma_device(e_dev);

	e_dev_info = e_dev;



	ret = dma_async_device_register(&e_dev->dma_device);

	if (ret) {

drivers/platform/msm/ep_pcie/ep_pcie_com.h

+1 −10
Original line number Diff line number Diff line 
/* Copyright (c) 2015-2019, The Linux Foundation. All rights reserved.

/* Copyright (c) 2015-2020, The Linux Foundation. All rights reserved.

 *

 * This program is free software; you can redistribute it and/or modify

 * it under the terms of the GNU General Public License version 2 and
@@ -420,15 +420,6 @@ struct ep_pcie_dev_t { 
extern struct ep_pcie_dev_t ep_pcie_dev;

extern struct ep_pcie_hw hw_drv;



#if IS_ENABLED(CONFIG_QCOM_PCI_EDMA)

int qcom_edma_init(struct device *dev);

#else

static inline int qcom_edma_init(struct device *dev)

{

	return 0;

}

#endif



static inline void ep_pcie_write_mask(void __iomem *addr,

				u32 clear_mask, u32 set_mask)

{

drivers/platform/msm/ep_pcie/ep_pcie_dbg.c

+4 −1
Original line number Diff line number Diff line 
/* Copyright (c) 2015-2019, The Linux Foundation. All rights reserved.

/* Copyright (c) 2015-2020, The Linux Foundation. All rights reserved.

 *

 * This program is free software; you can redistribute it and/or modify

 * it under the terms of the GNU General Public License version 2 and
@@ -406,6 +406,9 @@ static ssize_t ep_pcie_cmd_debug(struct file *file, 
	case 32: /* do not output core registers when D3 hot is set by host*/

		dev->dump_conf = false;

		break;

	case 33: /* output edma registers */

		edma_dump();

		break;

	default:

		EP_PCIE_DBG_FS("PCIe: Invalid testcase: %d\n", testcase);

		break;

include/linux/msm_ep_pcie.h

+13 −1
Original line number Diff line number Diff line 
/* Copyright (c) 2015, 2017, 2019, The Linux Foundation. All rights reserved.

/* Copyright (c) 2015, 2017, 2019-2020, The Linux Foundation. All rights reserved.

 *

 * This program is free software; you can redistribute it and/or modify

 * it under the terms of the GNU General Public License version 2 and
@@ -316,4 +316,16 @@ int ep_pcie_configure_inactivity_timer(struct ep_pcie_hw *phandle, 
 * Return: 0 on success, negative value on error

 */

int ep_pcie_core_l1ss_sleep_config_enable(void);



#if IS_ENABLED(CONFIG_QCOM_PCI_EDMA)

int qcom_edma_init(struct device *dev);

void edma_dump(void);

#else

static inline int qcom_edma_init(struct device *dev)

{

	return 0;

}

static inline void edma_dump(void) {}

#endif



#endif