Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 5086924a authored by Hans Verkuil's avatar Hans Verkuil Committed by Mauro Carvalho Chehab
Browse files

[media] vivid: fix try_fmt behavior 



vivid_try_fmt_vid_cap() called tpg_calc_line_width to calculate the sizeimage
value, but that tpg function uses the current format, not the proposed (tried)
format.

Rewrote this code to calculate this correctly.

The vivid_try_fmt_vid_out() code was completely wrong w.r.t. sizeimage, and
neither did it take the vdownsampling[] factors into account.

Signed-off-by: default avatarHans Verkuil <hans.verkuil@cisco.com>
Tested-by: default avatarVincent Abriou <vincent.abriou@st.com>
Signed-off-by: default avatarMauro Carvalho Chehab <mchehab@s-opensource.com>
parent 733e009c

drivers/media/platform/vivid/vivid-vid-cap.c

+10 −3
Original line number Diff line number Diff line
@@ -616,7 +616,7 @@ int vivid_try_fmt_vid_cap(struct file *file, void *priv, 
	/* This driver supports custom bytesperline values */



	mp->num_planes = fmt->buffers;

	for (p = 0; p < mp->num_planes; p++) {

	for (p = 0; p < fmt->buffers; p++) {

		/* Calculate the minimum supported bytesperline value */

		bytesperline = (mp->width * fmt->bit_depth[p]) >> 3;

		/* Calculate the maximum supported bytesperline value */
@@ -626,10 +626,17 @@ int vivid_try_fmt_vid_cap(struct file *file, void *priv, 
			pfmt[p].bytesperline = max_bpl;

		if (pfmt[p].bytesperline < bytesperline)

			pfmt[p].bytesperline = bytesperline;

		pfmt[p].sizeimage = tpg_calc_line_width(&dev->tpg, p, pfmt[p].bytesperline) *

			mp->height + fmt->data_offset[p];



		pfmt[p].sizeimage = (pfmt[p].bytesperline * mp->height) /

				fmt->vdownsampling[p] + fmt->data_offset[p];



		memset(pfmt[p].reserved, 0, sizeof(pfmt[p].reserved));

	}

	for (p = fmt->buffers; p < fmt->planes; p++)

		pfmt[0].sizeimage += (pfmt[0].bytesperline * mp->height *

			(fmt->bit_depth[p] / fmt->vdownsampling[p])) /

			(fmt->bit_depth[0] / fmt->vdownsampling[0]);



	mp->colorspace = vivid_colorspace_cap(dev);

	if (fmt->color_enc == TGP_COLOR_ENC_HSV)

		mp->hsv_enc = vivid_hsv_enc_cap(dev);

drivers/media/platform/vivid/vivid-vid-out.c

+14 −8
Original line number Diff line number Diff line
@@ -390,22 +390,28 @@ int vivid_try_fmt_vid_out(struct file *file, void *priv, 


	/* This driver supports custom bytesperline values */



	mp->num_planes = fmt->buffers;

	for (p = 0; p < fmt->buffers; p++) {

		/* Calculate the minimum supported bytesperline value */

	bytesperline = (mp->width * fmt->bit_depth[0]) >> 3;

		bytesperline = (mp->width * fmt->bit_depth[p]) >> 3;

		/* Calculate the maximum supported bytesperline value */

	max_bpl = (MAX_ZOOM * MAX_WIDTH * fmt->bit_depth[0]) >> 3;

	mp->num_planes = fmt->buffers;

	for (p = 0; p < mp->num_planes; p++) {

		max_bpl = (MAX_ZOOM * MAX_WIDTH * fmt->bit_depth[p]) >> 3;



		if (pfmt[p].bytesperline > max_bpl)

			pfmt[p].bytesperline = max_bpl;

		if (pfmt[p].bytesperline < bytesperline)

			pfmt[p].bytesperline = bytesperline;

		pfmt[p].sizeimage = pfmt[p].bytesperline * mp->height;



		pfmt[p].sizeimage = (pfmt[p].bytesperline * mp->height) /

					fmt->vdownsampling[p];



		memset(pfmt[p].reserved, 0, sizeof(pfmt[p].reserved));

	}

	for (p = fmt->buffers; p < fmt->planes; p++)

		pfmt[0].sizeimage += (pfmt[0].bytesperline * fmt->bit_depth[p]) /

				     (fmt->bit_depth[0] * fmt->vdownsampling[p]);

		pfmt[0].sizeimage += (pfmt[0].bytesperline * mp->height *

			(fmt->bit_depth[p] / fmt->vdownsampling[p])) /

			(fmt->bit_depth[0] / fmt->vdownsampling[0]);



	mp->xfer_func = V4L2_XFER_FUNC_DEFAULT;

	mp->ycbcr_enc = V4L2_YCBCR_ENC_DEFAULT;

	mp->quantization = V4L2_QUANTIZATION_DEFAULT;