Loading drivers/video/msm/mdss/mdp3.c +356 −17 Original line number Diff line number Diff line Loading @@ -15,6 +15,7 @@ #include <linux/clk.h> #include <linux/debugfs.h> #include <linux/dma-buf.h> #include <linux/kernel.h> #include <linux/init.h> #include <linux/interrupt.h> Loading Loading @@ -123,8 +124,19 @@ struct mdp3_bus_handle_map mdp3_bus_handle[MDP3_BUS_HANDLE_MAX] = { }; struct mdp3_iommu_domain_map mdp3_iommu_domains[MDP3_IOMMU_DOMAIN_MAX] = { [MDP3_IOMMU_DOMAIN] = { .domain_type = MDP3_IOMMU_DOMAIN, [MDP3_PPP_IOMMU_DOMAIN] = { .domain_type = MDP3_PPP_IOMMU_DOMAIN, .client_name = "mdp_ppp", .partitions = { { .start = SZ_128K, .size = SZ_1G - SZ_128K, }, }, .npartitions = 1, }, [MDP3_DMA_IOMMU_DOMAIN] = { .domain_type = MDP3_DMA_IOMMU_DOMAIN, .client_name = "mdp_dma", .partitions = { { Loading @@ -139,27 +151,27 @@ struct mdp3_iommu_domain_map mdp3_iommu_domains[MDP3_IOMMU_DOMAIN_MAX] = { struct mdp3_iommu_ctx_map mdp3_iommu_contexts[MDP3_IOMMU_CTX_MAX] = { [MDP3_IOMMU_CTX_PPP_0] = { .ctx_type = MDP3_IOMMU_CTX_PPP_0, .domain = &mdp3_iommu_domains[MDP3_IOMMU_DOMAIN], .domain = &mdp3_iommu_domains[MDP3_PPP_IOMMU_DOMAIN], .ctx_name = "mdpe_0", .attached = 0, }, [MDP3_IOMMU_CTX_PPP_1] = { .ctx_type = MDP3_IOMMU_CTX_PPP_1, .domain = &mdp3_iommu_domains[MDP3_IOMMU_DOMAIN], .domain = &mdp3_iommu_domains[MDP3_PPP_IOMMU_DOMAIN], .ctx_name = "mdpe_1", .attached = 0, }, [MDP3_IOMMU_CTX_DMA_0] = { .ctx_type = MDP3_IOMMU_CTX_DMA_0, .domain = &mdp3_iommu_domains[MDP3_IOMMU_DOMAIN], .domain = &mdp3_iommu_domains[MDP3_DMA_IOMMU_DOMAIN], .ctx_name = "mdps_0", .attached = 0, }, [MDP3_IOMMU_CTX_DMA_1] = { .ctx_type = MDP3_IOMMU_CTX_DMA_1, .domain = &mdp3_iommu_domains[MDP3_IOMMU_DOMAIN], .domain = &mdp3_iommu_domains[MDP3_DMA_IOMMU_DOMAIN], .ctx_name = "mdps_1", .attached = 0, }, Loading Loading @@ -688,6 +700,8 @@ int mdp3_iommu_init(void) { int ret; mutex_init(&mdp3_res->iommu_lock); ret = mdp3_iommu_domain_init(); if (ret) { pr_err("mdp3 iommu domain init fails\n"); Loading Loading @@ -1035,17 +1049,336 @@ static int mdp3_parse_dt(struct platform_device *pdev) return 0; } int mdp3_put_img(struct mdp3_img_data *data) static void mdp3_iommu_heap_unmap_iommu(struct mdp3_iommu_meta *meta) { unsigned int domain_num; unsigned int partition_num = 0; struct iommu_domain *domain; domain_num = (mdp3_res->domains + MDP3_PPP_IOMMU_DOMAIN)->domain_idx; domain = msm_get_iommu_domain(domain_num); if (!domain) { pr_err("Could not get domain %d. Corruption?\n", domain_num); return; } iommu_unmap_range(domain, meta->iova_addr, meta->mapped_size); msm_free_iova_address(meta->iova_addr, domain_num, partition_num, meta->mapped_size); return; } static void mdp3_iommu_meta_destroy(struct kref *kref) { struct mdp3_iommu_meta *meta = container_of(kref, struct mdp3_iommu_meta, ref); rb_erase(&meta->node, &mdp3_res->iommu_root); mdp3_iommu_heap_unmap_iommu(meta); dma_buf_put(meta->dbuf); kfree(meta); } static void mdp3_iommu_meta_put(struct mdp3_iommu_meta *meta) { /* Need to lock here to prevent race against map/unmap */ mutex_lock(&mdp3_res->iommu_lock); kref_put(&meta->ref, mdp3_iommu_meta_destroy); mutex_unlock(&mdp3_res->iommu_lock); } static struct mdp3_iommu_meta *mdp3_iommu_meta_lookup(struct sg_table *table) { struct rb_root *root = &mdp3_res->iommu_root; struct rb_node **p = &root->rb_node; struct rb_node *parent = NULL; struct mdp3_iommu_meta *entry = NULL; while (*p) { parent = *p; entry = rb_entry(parent, struct mdp3_iommu_meta, node); if (table < entry->table) p = &(*p)->rb_left; else if (table > entry->table) p = &(*p)->rb_right; else return entry; } return NULL; } void mdp3_unmap_iommu(struct ion_client *client, struct ion_handle *handle) { struct mdp3_iommu_meta *meta; struct sg_table *table; table = ion_sg_table(client, handle); mutex_lock(&mdp3_res->iommu_lock); meta = mdp3_iommu_meta_lookup(table); if (!meta) { WARN(1, "%s: buffer was never mapped for %p\n", __func__, handle); mutex_unlock(&mdp3_res->iommu_lock); goto out; } mutex_unlock(&mdp3_res->iommu_lock); mdp3_iommu_meta_put(meta); out: return; } static void mdp3_iommu_meta_add(struct mdp3_iommu_meta *meta) { struct rb_root *root = &mdp3_res->iommu_root; struct rb_node **p = &root->rb_node; struct rb_node *parent = NULL; struct mdp3_iommu_meta *entry; while (*p) { parent = *p; entry = rb_entry(parent, struct mdp3_iommu_meta, node); if (meta->table < entry->table) { p = &(*p)->rb_left; } else if (meta->table > entry->table) { p = &(*p)->rb_right; } else { pr_err("%s: handle %p already exists\n", __func__, entry->handle); BUG(); } } rb_link_node(&meta->node, parent, p); rb_insert_color(&meta->node, root); } static int mdp3_iommu_map_iommu(struct mdp3_iommu_meta *meta, unsigned long align, unsigned long iova_length, unsigned int padding, unsigned long flags) { struct iommu_domain *domain; int ret = 0; unsigned long size; unsigned long unmap_size; struct sg_table *table; int prot = IOMMU_WRITE | IOMMU_READ; unsigned int domain_num = (mdp3_res->domains + MDP3_PPP_IOMMU_DOMAIN)->domain_idx; unsigned int partition_num = 0; size = meta->size; table = meta->table; /* Use the biggest alignment to allow bigger IOMMU mappings. * Use the first entry since the first entry will always be the * biggest entry. To take advantage of bigger mapping sizes both the * VA and PA addresses have to be aligned to the biggest size. */ if (sg_dma_len(table->sgl) > align) align = sg_dma_len(table->sgl); ret = msm_allocate_iova_address(domain_num, partition_num, meta->mapped_size, align, (unsigned long *)&meta->iova_addr); if (ret) goto out; domain = msm_get_iommu_domain(domain_num); if (!domain) { ret = -ENOMEM; goto out1; } /* Adding padding to before buffer */ if (padding) { unsigned long phys_addr = sg_phys(table->sgl); ret = msm_iommu_map_extra(domain, meta->iova_addr, phys_addr, padding, SZ_4K, prot); if (ret) goto out1; } /* Mapping actual buffer */ ret = iommu_map_range(domain, meta->iova_addr + padding, table->sgl, size, prot); if (ret) { pr_err("%s: could not map %pa in domain %p\n", __func__, &meta->iova_addr, domain); unmap_size = padding; goto out2; } /* Adding padding to end of buffer */ if (padding) { unsigned long phys_addr = sg_phys(table->sgl); unsigned long extra_iova_addr = meta->iova_addr + padding + size; ret = msm_iommu_map_extra(domain, extra_iova_addr, phys_addr, padding, SZ_4K, prot); if (ret) { unmap_size = padding + size; goto out2; } } return ret; out2: iommu_unmap_range(domain, meta->iova_addr, unmap_size); out1: msm_free_iova_address(meta->iova_addr, domain_num, partition_num, iova_length); out: return ret; } static struct mdp3_iommu_meta *mdp3_iommu_meta_create(struct ion_client *client, struct ion_handle *handle, struct sg_table *table, unsigned long size, unsigned long align, unsigned long iova_length, unsigned int padding, unsigned long flags, dma_addr_t *iova) { struct mdp3_iommu_meta *meta; int ret; meta = kzalloc(sizeof(*meta), GFP_KERNEL); if (!meta) return ERR_PTR(-ENOMEM); meta->handle = handle; meta->table = table; meta->size = size; meta->mapped_size = iova_length; meta->dbuf = ion_share_dma_buf(client, handle); kref_init(&meta->ref); ret = mdp3_iommu_map_iommu(meta, align, iova_length, padding, flags); if (ret < 0) { pr_err("%s: Unable to map buffer\n", __func__); goto out; } *iova = meta->iova_addr; mdp3_iommu_meta_add(meta); return meta; out: kfree(meta); return ERR_PTR(ret); } /* * PPP hw reads in tiles of 16 which might be outside mapped region * need to map buffers ourseleve to add extra padding */ int mdp3_self_map_iommu(struct ion_client *client, struct ion_handle *handle, unsigned long align, unsigned long padding, dma_addr_t *iova, unsigned long *buffer_size, unsigned long flags, unsigned long iommu_flags) { struct mdp3_iommu_meta *iommu_meta = NULL; struct sg_table *table; struct scatterlist *sg; unsigned long size = 0, iova_length = 0; int ret = 0; int i; table = ion_sg_table(client, handle); if (IS_ERR_OR_NULL(table)) return PTR_ERR(table); for_each_sg(table->sgl, sg, table->nents, i) size += sg_dma_len(sg); padding = PAGE_ALIGN(padding); /* Adding 16 lines padding before and after buffer */ iova_length = size + 2 * padding; if (size & ~PAGE_MASK) { pr_debug("%s: buffer size %lx is not aligned to %lx", __func__, size, PAGE_SIZE); ret = -EINVAL; goto out; } if (iova_length & ~PAGE_MASK) { pr_debug("%s: iova_length %lx is not aligned to %lx", __func__, iova_length, PAGE_SIZE); ret = -EINVAL; goto out; } mutex_lock(&mdp3_res->iommu_lock); iommu_meta = mdp3_iommu_meta_lookup(table); if (!iommu_meta) { iommu_meta = mdp3_iommu_meta_create(client, handle, table, size, align, iova_length, padding, flags, iova); if (!IS_ERR_OR_NULL(iommu_meta)) { iommu_meta->flags = iommu_flags; ret = 0; } else { ret = PTR_ERR(iommu_meta); goto out_unlock; } } else { if (iommu_meta->flags != iommu_flags) { pr_err("%s: handle %p is already mapped with diff flag\n", __func__, handle); ret = -EINVAL; goto out_unlock; } else if (iommu_meta->mapped_size != iova_length) { pr_err("%s: handle %p is already mapped with diff len\n", __func__, handle); ret = -EINVAL; goto out_unlock; } else { kref_get(&iommu_meta->ref); *iova = iommu_meta->iova_addr; } } BUG_ON(iommu_meta->size != size); mutex_unlock(&mdp3_res->iommu_lock); *iova = *iova + padding; *buffer_size = size; return ret; out_unlock: mutex_unlock(&mdp3_res->iommu_lock); out: mdp3_iommu_meta_put(iommu_meta); return ret; } int mdp3_put_img(struct mdp3_img_data *data, int client) { struct ion_client *iclient = mdp3_res->ion_client; int dom = (mdp3_res->domains + MDP3_IOMMU_DOMAIN)->domain_idx; int dom; if (data->flags & MDP_MEMORY_ID_TYPE_FB) { pr_info("mdp3_put_img fb mem buf=0x%pa\n", &data->addr); fput_light(data->srcp_file, data->p_need); data->srcp_file = NULL; } else if (!IS_ERR_OR_NULL(data->srcp_ihdl)) { if (client == MDP3_CLIENT_DMA_P) { dom = (mdp3_res->domains + MDP3_DMA_IOMMU_DOMAIN)->domain_idx; ion_unmap_iommu(iclient, data->srcp_ihdl, dom, 0); } else { mdp3_unmap_iommu(iclient, data->srcp_ihdl); } ion_free(iclient, data->srcp_ihdl); data->srcp_ihdl = NULL; } else { Loading @@ -1054,7 +1387,8 @@ int mdp3_put_img(struct mdp3_img_data *data) return 0; } int mdp3_get_img(struct msmfb_data *img, struct mdp3_img_data *data) int mdp3_get_img(struct msmfb_data *img, struct mdp3_img_data *data, int client) { struct file *file; int ret = -EINVAL; Loading @@ -1062,7 +1396,7 @@ int mdp3_get_img(struct msmfb_data *img, struct mdp3_img_data *data) unsigned long *len; dma_addr_t *start; struct ion_client *iclient = mdp3_res->ion_client; int dom = (mdp3_res->domains + MDP3_IOMMU_DOMAIN)->domain_idx; int dom; start = &data->addr; len = (unsigned long *) &data->len; Loading Loading @@ -1104,10 +1438,15 @@ int mdp3_get_img(struct msmfb_data *img, struct mdp3_img_data *data) data->srcp_ihdl = NULL; return ret; } if (client == MDP3_CLIENT_DMA_P) { dom = (mdp3_res->domains + MDP3_DMA_IOMMU_DOMAIN)->domain_idx; ret = ion_map_iommu(iclient, data->srcp_ihdl, dom, 0, SZ_4K, 0, start, len, 0, 0); } else { ret = mdp3_self_map_iommu(iclient, data->srcp_ihdl, SZ_4K, data->padding, start, len, 0, 0); } if (IS_ERR_VALUE(ret)) { ion_free(iclient, data->srcp_ihdl); pr_err("failed to map ion handle (%d)\n", ret); Loading @@ -1122,7 +1461,7 @@ done: pr_debug("mem=%d ihdl=%p buf=0x%pa len=0x%x\n", img->memory_id, data->srcp_ihdl, &data->addr, data->len); } else { mdp3_put_img(data); mdp3_put_img(data, client); return -EINVAL; } Loading Loading @@ -1331,7 +1670,7 @@ static int mdp3_fb_mem_get_iommu_domain(void) { if (!mdp3_res) return -ENODEV; return mdp3_res->domains[MDP3_IOMMU_DOMAIN].domain_idx; return mdp3_res->domains[MDP3_DMA_IOMMU_DOMAIN].domain_idx; } int mdp3_continuous_splash_copy(struct mdss_panel_data *pdata) Loading drivers/video/msm/mdss/mdp3.h +28 −4 Original line number Diff line number Diff line Loading @@ -40,8 +40,9 @@ enum { }; enum { MDP3_IOMMU_DOMAIN, MDP3_IOMMU_DOMAIN_MAX MDP3_DMA_IOMMU_DOMAIN, MDP3_PPP_IOMMU_DOMAIN, MDP3_IOMMU_DOMAIN_MAX, }; enum { Loading @@ -57,6 +58,12 @@ enum { MDP3_CLIENT_PPP, }; enum { DI_PARTITION_NUM = 0, DI_DOMAIN_NUM = 1, DI_MAX, }; struct mdp3_bus_handle_map { struct msm_bus_vectors *bus_vector; struct msm_bus_paths *usecases; Loading @@ -82,6 +89,19 @@ struct mdp3_iommu_ctx_map { int attached; }; struct mdp3_iommu_meta { struct rb_node node; struct ion_handle *handle; struct rb_root iommu_maps; struct kref ref; struct sg_table *table; struct dma_buf *dbuf; int mapped_size; unsigned long size; dma_addr_t iova_addr; unsigned long flags; }; #define MDP3_MAX_INTR 28 struct mdp3_intr_cb { Loading Loading @@ -110,6 +130,8 @@ struct mdp3_hw_resource { struct mdp3_iommu_domain_map *domains; struct mdp3_iommu_ctx_map *iommu_contexts; struct ion_handle *ion_handle; struct mutex iommu_lock; struct rb_root iommu_root; void *virt; unsigned long phys; size_t size; Loading @@ -131,6 +153,7 @@ struct mdp3_hw_resource { struct mdp3_img_data { dma_addr_t addr; u32 len; u32 padding; u32 flags; int p_need; struct file *srcp_file; Loading @@ -150,8 +173,9 @@ void mdp3_irq_deregister(void); int mdp3_clk_set_rate(int clk_type, unsigned long clk_rate, int client); int mdp3_clk_enable(int enable); int mdp3_bus_scale_set_quota(int client, u64 ab_quota, u64 ib_quota); int mdp3_put_img(struct mdp3_img_data *data); int mdp3_get_img(struct msmfb_data *img, struct mdp3_img_data *data); int mdp3_put_img(struct mdp3_img_data *data, int client); int mdp3_get_img(struct msmfb_data *img, struct mdp3_img_data *data, int client); int mdp3_iommu_enable(int client); int mdp3_iommu_disable(int client); int mdp3_iommu_is_attached(int client); Loading drivers/video/msm/mdss/mdp3_ctrl.c +4 −4 Original line number Diff line number Diff line Loading @@ -50,7 +50,7 @@ static void mdp3_bufq_deinit(struct mdp3_buffer_queue *bufq) while (count--) { struct mdp3_img_data *data = &bufq->img_data[bufq->pop_idx]; bufq->pop_idx = (bufq->pop_idx + 1) % MDP3_MAX_BUF_QUEUE; mdp3_put_img(data); mdp3_put_img(data, MDP3_CLIENT_DMA_P); } bufq->count = 0; bufq->push_idx = 0; Loading Loading @@ -706,7 +706,7 @@ static int mdp3_overlay_queue_buffer(struct msm_fb_data_type *mfd, struct msmfb_data *img = &req->data; struct mdp3_img_data data; rc = mdp3_get_img(img, &data); rc = mdp3_get_img(img, &data, MDP3_CLIENT_DMA_P); if (rc) { pr_err("fail to get overlay buffer\n"); return rc; Loading @@ -715,7 +715,7 @@ static int mdp3_overlay_queue_buffer(struct msm_fb_data_type *mfd, rc = mdp3_bufq_push(&mdp3_session->bufq_in, &data); if (rc) { pr_err("fail to queue the overlay buffer, buffer drop\n"); mdp3_put_img(&data); mdp3_put_img(&data, MDP3_CLIENT_DMA_P); return rc; } return 0; Loading Loading @@ -779,7 +779,7 @@ static int mdp3_ctrl_display_commit_kickoff(struct msm_fb_data_type *mfd) if (mdp3_bufq_count(&mdp3_session->bufq_out) > 2) { data = mdp3_bufq_pop(&mdp3_session->bufq_out); mdp3_put_img(data); mdp3_put_img(data, MDP3_CLIENT_DMA_P); } mutex_unlock(&mdp3_session->lock); Loading drivers/video/msm/mdss/mdp3_ppp.c +19 −6 Original line number Diff line number Diff line Loading @@ -119,11 +119,22 @@ int mdp3_ppp_get_img(struct mdp_img *img, struct mdp_blit_req *req, struct mdp3_img_data *data) { struct msmfb_data fb_data; uint32_t stride; int bpp = ppp_bpp(img->format); if (bpp <= 0) { pr_err("%s incorrect format %d\n", __func__, img->format); return -EINVAL; } fb_data.flags = img->priv; fb_data.memory_id = img->memory_id; fb_data.offset = 0; return mdp3_get_img(&fb_data, data); stride = img->width * bpp; data->padding = 16 * stride; return mdp3_get_img(&fb_data, data, MDP3_CLIENT_PPP); } /* Check format */ Loading Loading @@ -945,8 +956,10 @@ static void mdp3_ppp_blit_wq_handler(struct work_struct *work) &req->src_data[i], &req->dst_data[i]); } mdp3_put_img(&req->src_data[i]); mdp3_put_img(&req->dst_data[i]); mdp3_put_img(&req->src_data[i], MDP3_CLIENT_PPP); mdp3_put_img(&req->dst_data[i], MDP3_CLIENT_PPP); } } /* Signal to release fence */ Loading Loading @@ -1017,7 +1030,7 @@ int mdp3_ppp_parse_req(void __user *p, rc = mdp3_ppp_get_img(&req->req_list[i].dst, &req->req_list[i], &req->dst_data[i]); if (rc < 0 || req->dst_data[i].len == 0) { mdp3_put_img(&req->src_data[i]); mdp3_put_img(&req->src_data[i], MDP3_CLIENT_PPP); pr_err("mdp_ppp: couldn't retrieve dest img from mem\n"); goto parse_err_1; } Loading Loading @@ -1060,8 +1073,8 @@ parse_err_2: put_unused_fd(req->cur_rel_fen_fd); parse_err_1: for (i--; i >= 0; i--) { mdp3_put_img(&req->src_data[i]); mdp3_put_img(&req->dst_data[i]); mdp3_put_img(&req->src_data[i], MDP3_CLIENT_PPP); mdp3_put_img(&req->dst_data[i], MDP3_CLIENT_PPP); } mdp3_ppp_deinit_buf_sync(req); mutex_unlock(&ppp_stat->req_mutex); Loading drivers/video/msm/mdss/mdp3_ppp_hwio.c +2 −40 Original line number Diff line number Diff line Loading @@ -361,38 +361,6 @@ bool check_if_rgb(int color) return rgb; } uint8_t *mdp_bg_adjust_rot_addr(struct ppp_blit_op *iBuf, uint8_t *addr, uint32_t bpp, uint32_t uv) { uint32_t dest_ystride = iBuf->bg.prop.width * bpp; uint32_t h_slice = 1, min_val; if (uv && ((iBuf->bg.color_fmt == MDP_Y_CBCR_H2V2) || (iBuf->bg.color_fmt == MDP_Y_CRCB_H2V2))) h_slice = 2; if (((iBuf->mdp_op & MDPOP_ROT90) == MDPOP_ROT90) ^ ((iBuf->mdp_op & MDPOP_LR) == MDPOP_LR)) { min_val = (iBuf->bg.roi.width + iBuf->bg.roi.x) % 16; if (!min_val) min_val = 16; addr += (iBuf->bg.roi.width - MIN(min_val, iBuf->bg.roi.width)) * bpp; } if ((iBuf->mdp_op & MDPOP_UD) == MDPOP_UD) { min_val = (iBuf->bg.roi.height + iBuf->bg.roi.y) % 16; if (!min_val) min_val = 16; addr += ((iBuf->bg.roi.height - MIN(min_val, iBuf->bg.roi.height))/h_slice) * dest_ystride; } return addr; } uint8_t *mdp_dst_adjust_rot_addr(struct ppp_blit_op *iBuf, uint8_t *addr, uint32_t bpp, uint32_t uv) { Loading Loading @@ -434,9 +402,7 @@ void mdp_adjust_start_addr(struct ppp_blit_op *blit_op, img->p0 += (x + y * ALIGN(width, 128)) * bpp; else img->p0 += (x + y * width) * bpp; if (layer == 1) img->p0 = mdp_bg_adjust_rot_addr(blit_op, img->p0, bpp, 0); else if (layer == 2) if (layer != 0) img->p0 = mdp_dst_adjust_rot_addr(blit_op, img->p0, bpp, 0); if (img->p1) { Loading @@ -454,15 +420,11 @@ void mdp_adjust_start_addr(struct ppp_blit_op *blit_op, img->p1 += ((x / h_slice) * h_slice + ((y == 0) ? 0 : ((y + 1) / v_slice - 1) * width)) * bpp; if (layer == 1) { img->p0 = mdp_bg_adjust_rot_addr(blit_op, img->p0, bpp, 0); } else if (layer == 2) { if (layer != 0) img->p0 = mdp_dst_adjust_rot_addr(blit_op, img->p0, bpp, 0); } } } int load_ppp_lut(int tableType, uint32_t *lut) { Loading Loading
drivers/video/msm/mdss/mdp3.c +356 −17 Original line number Diff line number Diff line Loading @@ -15,6 +15,7 @@ #include <linux/clk.h> #include <linux/debugfs.h> #include <linux/dma-buf.h> #include <linux/kernel.h> #include <linux/init.h> #include <linux/interrupt.h> Loading Loading @@ -123,8 +124,19 @@ struct mdp3_bus_handle_map mdp3_bus_handle[MDP3_BUS_HANDLE_MAX] = { }; struct mdp3_iommu_domain_map mdp3_iommu_domains[MDP3_IOMMU_DOMAIN_MAX] = { [MDP3_IOMMU_DOMAIN] = { .domain_type = MDP3_IOMMU_DOMAIN, [MDP3_PPP_IOMMU_DOMAIN] = { .domain_type = MDP3_PPP_IOMMU_DOMAIN, .client_name = "mdp_ppp", .partitions = { { .start = SZ_128K, .size = SZ_1G - SZ_128K, }, }, .npartitions = 1, }, [MDP3_DMA_IOMMU_DOMAIN] = { .domain_type = MDP3_DMA_IOMMU_DOMAIN, .client_name = "mdp_dma", .partitions = { { Loading @@ -139,27 +151,27 @@ struct mdp3_iommu_domain_map mdp3_iommu_domains[MDP3_IOMMU_DOMAIN_MAX] = { struct mdp3_iommu_ctx_map mdp3_iommu_contexts[MDP3_IOMMU_CTX_MAX] = { [MDP3_IOMMU_CTX_PPP_0] = { .ctx_type = MDP3_IOMMU_CTX_PPP_0, .domain = &mdp3_iommu_domains[MDP3_IOMMU_DOMAIN], .domain = &mdp3_iommu_domains[MDP3_PPP_IOMMU_DOMAIN], .ctx_name = "mdpe_0", .attached = 0, }, [MDP3_IOMMU_CTX_PPP_1] = { .ctx_type = MDP3_IOMMU_CTX_PPP_1, .domain = &mdp3_iommu_domains[MDP3_IOMMU_DOMAIN], .domain = &mdp3_iommu_domains[MDP3_PPP_IOMMU_DOMAIN], .ctx_name = "mdpe_1", .attached = 0, }, [MDP3_IOMMU_CTX_DMA_0] = { .ctx_type = MDP3_IOMMU_CTX_DMA_0, .domain = &mdp3_iommu_domains[MDP3_IOMMU_DOMAIN], .domain = &mdp3_iommu_domains[MDP3_DMA_IOMMU_DOMAIN], .ctx_name = "mdps_0", .attached = 0, }, [MDP3_IOMMU_CTX_DMA_1] = { .ctx_type = MDP3_IOMMU_CTX_DMA_1, .domain = &mdp3_iommu_domains[MDP3_IOMMU_DOMAIN], .domain = &mdp3_iommu_domains[MDP3_DMA_IOMMU_DOMAIN], .ctx_name = "mdps_1", .attached = 0, }, Loading Loading @@ -688,6 +700,8 @@ int mdp3_iommu_init(void) { int ret; mutex_init(&mdp3_res->iommu_lock); ret = mdp3_iommu_domain_init(); if (ret) { pr_err("mdp3 iommu domain init fails\n"); Loading Loading @@ -1035,17 +1049,336 @@ static int mdp3_parse_dt(struct platform_device *pdev) return 0; } int mdp3_put_img(struct mdp3_img_data *data) static void mdp3_iommu_heap_unmap_iommu(struct mdp3_iommu_meta *meta) { unsigned int domain_num; unsigned int partition_num = 0; struct iommu_domain *domain; domain_num = (mdp3_res->domains + MDP3_PPP_IOMMU_DOMAIN)->domain_idx; domain = msm_get_iommu_domain(domain_num); if (!domain) { pr_err("Could not get domain %d. Corruption?\n", domain_num); return; } iommu_unmap_range(domain, meta->iova_addr, meta->mapped_size); msm_free_iova_address(meta->iova_addr, domain_num, partition_num, meta->mapped_size); return; } static void mdp3_iommu_meta_destroy(struct kref *kref) { struct mdp3_iommu_meta *meta = container_of(kref, struct mdp3_iommu_meta, ref); rb_erase(&meta->node, &mdp3_res->iommu_root); mdp3_iommu_heap_unmap_iommu(meta); dma_buf_put(meta->dbuf); kfree(meta); } static void mdp3_iommu_meta_put(struct mdp3_iommu_meta *meta) { /* Need to lock here to prevent race against map/unmap */ mutex_lock(&mdp3_res->iommu_lock); kref_put(&meta->ref, mdp3_iommu_meta_destroy); mutex_unlock(&mdp3_res->iommu_lock); } static struct mdp3_iommu_meta *mdp3_iommu_meta_lookup(struct sg_table *table) { struct rb_root *root = &mdp3_res->iommu_root; struct rb_node **p = &root->rb_node; struct rb_node *parent = NULL; struct mdp3_iommu_meta *entry = NULL; while (*p) { parent = *p; entry = rb_entry(parent, struct mdp3_iommu_meta, node); if (table < entry->table) p = &(*p)->rb_left; else if (table > entry->table) p = &(*p)->rb_right; else return entry; } return NULL; } void mdp3_unmap_iommu(struct ion_client *client, struct ion_handle *handle) { struct mdp3_iommu_meta *meta; struct sg_table *table; table = ion_sg_table(client, handle); mutex_lock(&mdp3_res->iommu_lock); meta = mdp3_iommu_meta_lookup(table); if (!meta) { WARN(1, "%s: buffer was never mapped for %p\n", __func__, handle); mutex_unlock(&mdp3_res->iommu_lock); goto out; } mutex_unlock(&mdp3_res->iommu_lock); mdp3_iommu_meta_put(meta); out: return; } static void mdp3_iommu_meta_add(struct mdp3_iommu_meta *meta) { struct rb_root *root = &mdp3_res->iommu_root; struct rb_node **p = &root->rb_node; struct rb_node *parent = NULL; struct mdp3_iommu_meta *entry; while (*p) { parent = *p; entry = rb_entry(parent, struct mdp3_iommu_meta, node); if (meta->table < entry->table) { p = &(*p)->rb_left; } else if (meta->table > entry->table) { p = &(*p)->rb_right; } else { pr_err("%s: handle %p already exists\n", __func__, entry->handle); BUG(); } } rb_link_node(&meta->node, parent, p); rb_insert_color(&meta->node, root); } static int mdp3_iommu_map_iommu(struct mdp3_iommu_meta *meta, unsigned long align, unsigned long iova_length, unsigned int padding, unsigned long flags) { struct iommu_domain *domain; int ret = 0; unsigned long size; unsigned long unmap_size; struct sg_table *table; int prot = IOMMU_WRITE | IOMMU_READ; unsigned int domain_num = (mdp3_res->domains + MDP3_PPP_IOMMU_DOMAIN)->domain_idx; unsigned int partition_num = 0; size = meta->size; table = meta->table; /* Use the biggest alignment to allow bigger IOMMU mappings. * Use the first entry since the first entry will always be the * biggest entry. To take advantage of bigger mapping sizes both the * VA and PA addresses have to be aligned to the biggest size. */ if (sg_dma_len(table->sgl) > align) align = sg_dma_len(table->sgl); ret = msm_allocate_iova_address(domain_num, partition_num, meta->mapped_size, align, (unsigned long *)&meta->iova_addr); if (ret) goto out; domain = msm_get_iommu_domain(domain_num); if (!domain) { ret = -ENOMEM; goto out1; } /* Adding padding to before buffer */ if (padding) { unsigned long phys_addr = sg_phys(table->sgl); ret = msm_iommu_map_extra(domain, meta->iova_addr, phys_addr, padding, SZ_4K, prot); if (ret) goto out1; } /* Mapping actual buffer */ ret = iommu_map_range(domain, meta->iova_addr + padding, table->sgl, size, prot); if (ret) { pr_err("%s: could not map %pa in domain %p\n", __func__, &meta->iova_addr, domain); unmap_size = padding; goto out2; } /* Adding padding to end of buffer */ if (padding) { unsigned long phys_addr = sg_phys(table->sgl); unsigned long extra_iova_addr = meta->iova_addr + padding + size; ret = msm_iommu_map_extra(domain, extra_iova_addr, phys_addr, padding, SZ_4K, prot); if (ret) { unmap_size = padding + size; goto out2; } } return ret; out2: iommu_unmap_range(domain, meta->iova_addr, unmap_size); out1: msm_free_iova_address(meta->iova_addr, domain_num, partition_num, iova_length); out: return ret; } static struct mdp3_iommu_meta *mdp3_iommu_meta_create(struct ion_client *client, struct ion_handle *handle, struct sg_table *table, unsigned long size, unsigned long align, unsigned long iova_length, unsigned int padding, unsigned long flags, dma_addr_t *iova) { struct mdp3_iommu_meta *meta; int ret; meta = kzalloc(sizeof(*meta), GFP_KERNEL); if (!meta) return ERR_PTR(-ENOMEM); meta->handle = handle; meta->table = table; meta->size = size; meta->mapped_size = iova_length; meta->dbuf = ion_share_dma_buf(client, handle); kref_init(&meta->ref); ret = mdp3_iommu_map_iommu(meta, align, iova_length, padding, flags); if (ret < 0) { pr_err("%s: Unable to map buffer\n", __func__); goto out; } *iova = meta->iova_addr; mdp3_iommu_meta_add(meta); return meta; out: kfree(meta); return ERR_PTR(ret); } /* * PPP hw reads in tiles of 16 which might be outside mapped region * need to map buffers ourseleve to add extra padding */ int mdp3_self_map_iommu(struct ion_client *client, struct ion_handle *handle, unsigned long align, unsigned long padding, dma_addr_t *iova, unsigned long *buffer_size, unsigned long flags, unsigned long iommu_flags) { struct mdp3_iommu_meta *iommu_meta = NULL; struct sg_table *table; struct scatterlist *sg; unsigned long size = 0, iova_length = 0; int ret = 0; int i; table = ion_sg_table(client, handle); if (IS_ERR_OR_NULL(table)) return PTR_ERR(table); for_each_sg(table->sgl, sg, table->nents, i) size += sg_dma_len(sg); padding = PAGE_ALIGN(padding); /* Adding 16 lines padding before and after buffer */ iova_length = size + 2 * padding; if (size & ~PAGE_MASK) { pr_debug("%s: buffer size %lx is not aligned to %lx", __func__, size, PAGE_SIZE); ret = -EINVAL; goto out; } if (iova_length & ~PAGE_MASK) { pr_debug("%s: iova_length %lx is not aligned to %lx", __func__, iova_length, PAGE_SIZE); ret = -EINVAL; goto out; } mutex_lock(&mdp3_res->iommu_lock); iommu_meta = mdp3_iommu_meta_lookup(table); if (!iommu_meta) { iommu_meta = mdp3_iommu_meta_create(client, handle, table, size, align, iova_length, padding, flags, iova); if (!IS_ERR_OR_NULL(iommu_meta)) { iommu_meta->flags = iommu_flags; ret = 0; } else { ret = PTR_ERR(iommu_meta); goto out_unlock; } } else { if (iommu_meta->flags != iommu_flags) { pr_err("%s: handle %p is already mapped with diff flag\n", __func__, handle); ret = -EINVAL; goto out_unlock; } else if (iommu_meta->mapped_size != iova_length) { pr_err("%s: handle %p is already mapped with diff len\n", __func__, handle); ret = -EINVAL; goto out_unlock; } else { kref_get(&iommu_meta->ref); *iova = iommu_meta->iova_addr; } } BUG_ON(iommu_meta->size != size); mutex_unlock(&mdp3_res->iommu_lock); *iova = *iova + padding; *buffer_size = size; return ret; out_unlock: mutex_unlock(&mdp3_res->iommu_lock); out: mdp3_iommu_meta_put(iommu_meta); return ret; } int mdp3_put_img(struct mdp3_img_data *data, int client) { struct ion_client *iclient = mdp3_res->ion_client; int dom = (mdp3_res->domains + MDP3_IOMMU_DOMAIN)->domain_idx; int dom; if (data->flags & MDP_MEMORY_ID_TYPE_FB) { pr_info("mdp3_put_img fb mem buf=0x%pa\n", &data->addr); fput_light(data->srcp_file, data->p_need); data->srcp_file = NULL; } else if (!IS_ERR_OR_NULL(data->srcp_ihdl)) { if (client == MDP3_CLIENT_DMA_P) { dom = (mdp3_res->domains + MDP3_DMA_IOMMU_DOMAIN)->domain_idx; ion_unmap_iommu(iclient, data->srcp_ihdl, dom, 0); } else { mdp3_unmap_iommu(iclient, data->srcp_ihdl); } ion_free(iclient, data->srcp_ihdl); data->srcp_ihdl = NULL; } else { Loading @@ -1054,7 +1387,8 @@ int mdp3_put_img(struct mdp3_img_data *data) return 0; } int mdp3_get_img(struct msmfb_data *img, struct mdp3_img_data *data) int mdp3_get_img(struct msmfb_data *img, struct mdp3_img_data *data, int client) { struct file *file; int ret = -EINVAL; Loading @@ -1062,7 +1396,7 @@ int mdp3_get_img(struct msmfb_data *img, struct mdp3_img_data *data) unsigned long *len; dma_addr_t *start; struct ion_client *iclient = mdp3_res->ion_client; int dom = (mdp3_res->domains + MDP3_IOMMU_DOMAIN)->domain_idx; int dom; start = &data->addr; len = (unsigned long *) &data->len; Loading Loading @@ -1104,10 +1438,15 @@ int mdp3_get_img(struct msmfb_data *img, struct mdp3_img_data *data) data->srcp_ihdl = NULL; return ret; } if (client == MDP3_CLIENT_DMA_P) { dom = (mdp3_res->domains + MDP3_DMA_IOMMU_DOMAIN)->domain_idx; ret = ion_map_iommu(iclient, data->srcp_ihdl, dom, 0, SZ_4K, 0, start, len, 0, 0); } else { ret = mdp3_self_map_iommu(iclient, data->srcp_ihdl, SZ_4K, data->padding, start, len, 0, 0); } if (IS_ERR_VALUE(ret)) { ion_free(iclient, data->srcp_ihdl); pr_err("failed to map ion handle (%d)\n", ret); Loading @@ -1122,7 +1461,7 @@ done: pr_debug("mem=%d ihdl=%p buf=0x%pa len=0x%x\n", img->memory_id, data->srcp_ihdl, &data->addr, data->len); } else { mdp3_put_img(data); mdp3_put_img(data, client); return -EINVAL; } Loading Loading @@ -1331,7 +1670,7 @@ static int mdp3_fb_mem_get_iommu_domain(void) { if (!mdp3_res) return -ENODEV; return mdp3_res->domains[MDP3_IOMMU_DOMAIN].domain_idx; return mdp3_res->domains[MDP3_DMA_IOMMU_DOMAIN].domain_idx; } int mdp3_continuous_splash_copy(struct mdss_panel_data *pdata) Loading
drivers/video/msm/mdss/mdp3.h +28 −4 Original line number Diff line number Diff line Loading @@ -40,8 +40,9 @@ enum { }; enum { MDP3_IOMMU_DOMAIN, MDP3_IOMMU_DOMAIN_MAX MDP3_DMA_IOMMU_DOMAIN, MDP3_PPP_IOMMU_DOMAIN, MDP3_IOMMU_DOMAIN_MAX, }; enum { Loading @@ -57,6 +58,12 @@ enum { MDP3_CLIENT_PPP, }; enum { DI_PARTITION_NUM = 0, DI_DOMAIN_NUM = 1, DI_MAX, }; struct mdp3_bus_handle_map { struct msm_bus_vectors *bus_vector; struct msm_bus_paths *usecases; Loading @@ -82,6 +89,19 @@ struct mdp3_iommu_ctx_map { int attached; }; struct mdp3_iommu_meta { struct rb_node node; struct ion_handle *handle; struct rb_root iommu_maps; struct kref ref; struct sg_table *table; struct dma_buf *dbuf; int mapped_size; unsigned long size; dma_addr_t iova_addr; unsigned long flags; }; #define MDP3_MAX_INTR 28 struct mdp3_intr_cb { Loading Loading @@ -110,6 +130,8 @@ struct mdp3_hw_resource { struct mdp3_iommu_domain_map *domains; struct mdp3_iommu_ctx_map *iommu_contexts; struct ion_handle *ion_handle; struct mutex iommu_lock; struct rb_root iommu_root; void *virt; unsigned long phys; size_t size; Loading @@ -131,6 +153,7 @@ struct mdp3_hw_resource { struct mdp3_img_data { dma_addr_t addr; u32 len; u32 padding; u32 flags; int p_need; struct file *srcp_file; Loading @@ -150,8 +173,9 @@ void mdp3_irq_deregister(void); int mdp3_clk_set_rate(int clk_type, unsigned long clk_rate, int client); int mdp3_clk_enable(int enable); int mdp3_bus_scale_set_quota(int client, u64 ab_quota, u64 ib_quota); int mdp3_put_img(struct mdp3_img_data *data); int mdp3_get_img(struct msmfb_data *img, struct mdp3_img_data *data); int mdp3_put_img(struct mdp3_img_data *data, int client); int mdp3_get_img(struct msmfb_data *img, struct mdp3_img_data *data, int client); int mdp3_iommu_enable(int client); int mdp3_iommu_disable(int client); int mdp3_iommu_is_attached(int client); Loading
drivers/video/msm/mdss/mdp3_ctrl.c +4 −4 Original line number Diff line number Diff line Loading @@ -50,7 +50,7 @@ static void mdp3_bufq_deinit(struct mdp3_buffer_queue *bufq) while (count--) { struct mdp3_img_data *data = &bufq->img_data[bufq->pop_idx]; bufq->pop_idx = (bufq->pop_idx + 1) % MDP3_MAX_BUF_QUEUE; mdp3_put_img(data); mdp3_put_img(data, MDP3_CLIENT_DMA_P); } bufq->count = 0; bufq->push_idx = 0; Loading Loading @@ -706,7 +706,7 @@ static int mdp3_overlay_queue_buffer(struct msm_fb_data_type *mfd, struct msmfb_data *img = &req->data; struct mdp3_img_data data; rc = mdp3_get_img(img, &data); rc = mdp3_get_img(img, &data, MDP3_CLIENT_DMA_P); if (rc) { pr_err("fail to get overlay buffer\n"); return rc; Loading @@ -715,7 +715,7 @@ static int mdp3_overlay_queue_buffer(struct msm_fb_data_type *mfd, rc = mdp3_bufq_push(&mdp3_session->bufq_in, &data); if (rc) { pr_err("fail to queue the overlay buffer, buffer drop\n"); mdp3_put_img(&data); mdp3_put_img(&data, MDP3_CLIENT_DMA_P); return rc; } return 0; Loading Loading @@ -779,7 +779,7 @@ static int mdp3_ctrl_display_commit_kickoff(struct msm_fb_data_type *mfd) if (mdp3_bufq_count(&mdp3_session->bufq_out) > 2) { data = mdp3_bufq_pop(&mdp3_session->bufq_out); mdp3_put_img(data); mdp3_put_img(data, MDP3_CLIENT_DMA_P); } mutex_unlock(&mdp3_session->lock); Loading
drivers/video/msm/mdss/mdp3_ppp.c +19 −6 Original line number Diff line number Diff line Loading @@ -119,11 +119,22 @@ int mdp3_ppp_get_img(struct mdp_img *img, struct mdp_blit_req *req, struct mdp3_img_data *data) { struct msmfb_data fb_data; uint32_t stride; int bpp = ppp_bpp(img->format); if (bpp <= 0) { pr_err("%s incorrect format %d\n", __func__, img->format); return -EINVAL; } fb_data.flags = img->priv; fb_data.memory_id = img->memory_id; fb_data.offset = 0; return mdp3_get_img(&fb_data, data); stride = img->width * bpp; data->padding = 16 * stride; return mdp3_get_img(&fb_data, data, MDP3_CLIENT_PPP); } /* Check format */ Loading Loading @@ -945,8 +956,10 @@ static void mdp3_ppp_blit_wq_handler(struct work_struct *work) &req->src_data[i], &req->dst_data[i]); } mdp3_put_img(&req->src_data[i]); mdp3_put_img(&req->dst_data[i]); mdp3_put_img(&req->src_data[i], MDP3_CLIENT_PPP); mdp3_put_img(&req->dst_data[i], MDP3_CLIENT_PPP); } } /* Signal to release fence */ Loading Loading @@ -1017,7 +1030,7 @@ int mdp3_ppp_parse_req(void __user *p, rc = mdp3_ppp_get_img(&req->req_list[i].dst, &req->req_list[i], &req->dst_data[i]); if (rc < 0 || req->dst_data[i].len == 0) { mdp3_put_img(&req->src_data[i]); mdp3_put_img(&req->src_data[i], MDP3_CLIENT_PPP); pr_err("mdp_ppp: couldn't retrieve dest img from mem\n"); goto parse_err_1; } Loading Loading @@ -1060,8 +1073,8 @@ parse_err_2: put_unused_fd(req->cur_rel_fen_fd); parse_err_1: for (i--; i >= 0; i--) { mdp3_put_img(&req->src_data[i]); mdp3_put_img(&req->dst_data[i]); mdp3_put_img(&req->src_data[i], MDP3_CLIENT_PPP); mdp3_put_img(&req->dst_data[i], MDP3_CLIENT_PPP); } mdp3_ppp_deinit_buf_sync(req); mutex_unlock(&ppp_stat->req_mutex); Loading
drivers/video/msm/mdss/mdp3_ppp_hwio.c +2 −40 Original line number Diff line number Diff line Loading @@ -361,38 +361,6 @@ bool check_if_rgb(int color) return rgb; } uint8_t *mdp_bg_adjust_rot_addr(struct ppp_blit_op *iBuf, uint8_t *addr, uint32_t bpp, uint32_t uv) { uint32_t dest_ystride = iBuf->bg.prop.width * bpp; uint32_t h_slice = 1, min_val; if (uv && ((iBuf->bg.color_fmt == MDP_Y_CBCR_H2V2) || (iBuf->bg.color_fmt == MDP_Y_CRCB_H2V2))) h_slice = 2; if (((iBuf->mdp_op & MDPOP_ROT90) == MDPOP_ROT90) ^ ((iBuf->mdp_op & MDPOP_LR) == MDPOP_LR)) { min_val = (iBuf->bg.roi.width + iBuf->bg.roi.x) % 16; if (!min_val) min_val = 16; addr += (iBuf->bg.roi.width - MIN(min_val, iBuf->bg.roi.width)) * bpp; } if ((iBuf->mdp_op & MDPOP_UD) == MDPOP_UD) { min_val = (iBuf->bg.roi.height + iBuf->bg.roi.y) % 16; if (!min_val) min_val = 16; addr += ((iBuf->bg.roi.height - MIN(min_val, iBuf->bg.roi.height))/h_slice) * dest_ystride; } return addr; } uint8_t *mdp_dst_adjust_rot_addr(struct ppp_blit_op *iBuf, uint8_t *addr, uint32_t bpp, uint32_t uv) { Loading Loading @@ -434,9 +402,7 @@ void mdp_adjust_start_addr(struct ppp_blit_op *blit_op, img->p0 += (x + y * ALIGN(width, 128)) * bpp; else img->p0 += (x + y * width) * bpp; if (layer == 1) img->p0 = mdp_bg_adjust_rot_addr(blit_op, img->p0, bpp, 0); else if (layer == 2) if (layer != 0) img->p0 = mdp_dst_adjust_rot_addr(blit_op, img->p0, bpp, 0); if (img->p1) { Loading @@ -454,15 +420,11 @@ void mdp_adjust_start_addr(struct ppp_blit_op *blit_op, img->p1 += ((x / h_slice) * h_slice + ((y == 0) ? 0 : ((y + 1) / v_slice - 1) * width)) * bpp; if (layer == 1) { img->p0 = mdp_bg_adjust_rot_addr(blit_op, img->p0, bpp, 0); } else if (layer == 2) { if (layer != 0) img->p0 = mdp_dst_adjust_rot_addr(blit_op, img->p0, bpp, 0); } } } int load_ppp_lut(int tableType, uint32_t *lut) { Loading