[libcamera-devel] [PATCH 7/7] qcam: viewfinder_gl: Add shader to render packed YUV formats
Niklas Söderlund
niklas.soderlund at ragnatech.se
Wed Sep 16 17:55:20 CEST 2020
Hi Laurent,
Thanks for your work.
On 2020-09-16 17:52:54 +0300, Laurent Pinchart wrote:
> The shader supports all 4 packed 8-bit YUV variants.
OpenGL is not my strong suite so I can't really give it a proper review.
>From what I do understand this change looks good but I'm unable to give
a R-b in good concious with my limited understanding. I do however think
the change is neatly modularized and properly tested with good results
so I think merging it adds value without any risk for regression or
later problems.
Acked-by: Niklas Söderlund <niklas.soderlund at ragnatech.se>
Please see bellow for a small typo tho.
>
> Signed-off-by: Laurent Pinchart <laurent.pinchart at ideasonboard.com>
> ---
> src/qcam/assets/shader/YUV_packed.frag | 82 ++++++++++++++++++++++++++
> src/qcam/assets/shader/shaders.qrc | 1 +
> src/qcam/viewfinder_gl.cpp | 56 ++++++++++++++++++
> src/qcam/viewfinder_gl.h | 1 +
> 4 files changed, 140 insertions(+)
> create mode 100644 src/qcam/assets/shader/YUV_packed.frag
>
> diff --git a/src/qcam/assets/shader/YUV_packed.frag b/src/qcam/assets/shader/YUV_packed.frag
> new file mode 100644
> index 000000000000..224dfafe383e
> --- /dev/null
> +++ b/src/qcam/assets/shader/YUV_packed.frag
> @@ -0,0 +1,82 @@
> +/* SPDX-License-Identifier: LGPL-2.1-or-later */
> +/*
> + * Copyright (C) 2020, Laurent Pinchart <laurent.pinchart at ideasonboard.com>
> + *
> + * YUV_packed.frag - Fragment shader code for YUYV packed formats
> + */
> +
> +#ifdef GL_ES
> +precision mediump float;
> +#endif
> +
> +varying vec2 textureOut;
> +
> +uniform sampler2D tex_y;
> +uniform float tex_stepx;
> +
> +void main(void)
> +{
> + mat3 yuv2rgb_bt601_mat = mat3(
> + vec3(1.164, 1.164, 1.164),
> + vec3(0.000, -0.392, 2.017),
> + vec3(1.596, -0.813, 0.000)
> + );
> + vec3 yuv2rgb_bt601_offset = vec3(0.063, 0.500, 0.500);
> +
> + /*
> + * The sampler won't interpolate the texture correctly along the X axis,
> + * as each RGBA pixel effectively stores two pixels. We thus need to
> + * interpolate manually.
> + *
> + * In integer texture coordinates, the Y values are layed out in the
> + * texture memory as follows:
> + *
> + * ...| Y U Y V | Y U Y V | Y U Y V |...
> + * ...| R G B A | R G B A | R G B A |...
> + * ^ ^ ^ ^ ^ ^
> + * | | | | | |
> + * n-1 n-0.5 n n+0.5 n+1 n+1.5
> + *
> + * For a texture location x in the interval [n, n+1[, sample the left
s/[n, n+1[/[n, n+1]/
> + * and right pixels at n and n+1, and interpolate them with
> + *
> + * left.r * (1 - a) + left.b * a if fract(x) < 0.5
> + * left.b * (1 - a) + right.r * a if fract(x) >= 0.5
> + *
> + * with a = fract(x * 2) which can also be written
> + *
> + * a = fract(x) * 2 if fract(x) < 0.5
> + * a = fract(x) * 2 - 1 if fract(x) >= 0.5
> + */
> + vec2 pos = textureOut;
> + float f_x = fract(pos.x / tex_stepx);
> +
> + vec4 left = texture2D(tex_y, vec2(pos.x - f_x * tex_stepx, pos.y));
> + vec4 right = texture2D(tex_y, vec2(pos.x + (1.0 - f_x) * tex_stepx , pos.y));
> +
> +#if defined(YUV_PATTERN_UYVY)
> + float y_left = mix(left.g, left.a, f_x * 2.0);
> + float y_right = mix(left.a, right.g, f_x * 2.0 - 1.0);
> + vec2 uv = mix(left.rb, right.rb, f_x);
> +#elif defined(YUV_PATTERN_VYUY)
> + float y_left = mix(left.g, left.a, f_x * 2.0);
> + float y_right = mix(left.a, right.g, f_x * 2.0 - 1.0);
> + vec2 uv = mix(left.br, right.br, f_x);
> +#elif defined(YUV_PATTERN_YUYV)
> + float y_left = mix(left.r, left.b, f_x * 2.0);
> + float y_right = mix(left.b, right.r, f_x * 2.0 - 1.0);
> + vec2 uv = mix(left.ga, right.ga, f_x);
> +#elif defined(YUV_PATTERN_YVYU)
> + float y_left = mix(left.r, left.b, f_x * 2.0);
> + float y_right = mix(left.b, right.r, f_x * 2.0 - 1.0);
> + vec2 uv = mix(left.ag, right.ag, f_x);
> +#else
> +#error Invalid pattern
> +#endif
> +
> + float y = mix(y_left, y_right, step(0.5, f_x));
> +
> + vec3 rgb = yuv2rgb_bt601_mat * (vec3(y, uv) - yuv2rgb_bt601_offset);
> +
> + gl_FragColor = vec4(rgb, 1.0);
> +}
> diff --git a/src/qcam/assets/shader/shaders.qrc b/src/qcam/assets/shader/shaders.qrc
> index 7010d8433c9b..857ed9fd5cde 100644
> --- a/src/qcam/assets/shader/shaders.qrc
> +++ b/src/qcam/assets/shader/shaders.qrc
> @@ -4,5 +4,6 @@
> <file>YUV.vert</file>
> <file>YUV_2_planes.frag</file>
> <file>YUV_3_planes.frag</file>
> + <file>YUV_packed.frag</file>
> </qresource>
> </RCC>
> diff --git a/src/qcam/viewfinder_gl.cpp b/src/qcam/viewfinder_gl.cpp
> index b8a4827267c3..0b5c942658cd 100644
> --- a/src/qcam/viewfinder_gl.cpp
> +++ b/src/qcam/viewfinder_gl.cpp
> @@ -14,12 +14,19 @@
> #include <libcamera/formats.h>
>
> static const QList<libcamera::PixelFormat> supportedFormats{
> + /* Packed (single plane) */
> + libcamera::formats::UYVY,
> + libcamera::formats::VYUY,
> + libcamera::formats::YUYV,
> + libcamera::formats::YVYU,
> + /* Semi planar (two planes) */
> libcamera::formats::NV12,
> libcamera::formats::NV21,
> libcamera::formats::NV16,
> libcamera::formats::NV61,
> libcamera::formats::NV24,
> libcamera::formats::NV42,
> + /* Fully planar (three planes) */
> libcamera::formats::YUV420,
> libcamera::formats::YVU420,
> };
> @@ -149,6 +156,22 @@ bool ViewFinderGL::selectFormat(const libcamera::PixelFormat &format)
> vertSubSample_ = 2;
> fragmentShaderFile_ = ":YUV_3_planes.frag";
> break;
> + case libcamera::formats::UYVY:
> + fragmentShaderDefines_.append("#define YUV_PATTERN_UYVY");
> + fragmentShaderFile_ = ":YUV_packed.frag";
> + break;
> + case libcamera::formats::VYUY:
> + fragmentShaderDefines_.append("#define YUV_PATTERN_VYUY");
> + fragmentShaderFile_ = ":YUV_packed.frag";
> + break;
> + case libcamera::formats::YUYV:
> + fragmentShaderDefines_.append("#define YUV_PATTERN_YUYV");
> + fragmentShaderFile_ = ":YUV_packed.frag";
> + break;
> + case libcamera::formats::YVYU:
> + fragmentShaderDefines_.append("#define YUV_PATTERN_YVYU");
> + fragmentShaderFile_ = ":YUV_packed.frag";
> + break;
> default:
> ret = false;
> qWarning() << "[ViewFinderGL]:"
> @@ -235,6 +258,7 @@ bool ViewFinderGL::createFragmentShader()
> textureUniformY_ = shaderProgram_.uniformLocation("tex_y");
> textureUniformU_ = shaderProgram_.uniformLocation("tex_u");
> textureUniformV_ = shaderProgram_.uniformLocation("tex_v");
> + textureUniformStepX_ = shaderProgram_.uniformLocation("tex_stepx");
>
> if (!textureY_.isCreated())
> textureY_.create();
> @@ -431,6 +455,38 @@ void ViewFinderGL::doRender()
> shaderProgram_.setUniformValue(textureUniformU_, 1);
> break;
>
> + case libcamera::formats::UYVY:
> + case libcamera::formats::VYUY:
> + case libcamera::formats::YUYV:
> + case libcamera::formats::YVYU:
> + /*
> + * Packed YUV formats are stored in a RGBA texture to match the
> + * OpenGL texel size with the 4 bytes repeating pattern in YUV.
> + * The texture width is thus half of the image with.
> + */
> + glActiveTexture(GL_TEXTURE0);
> + configureTexture(textureY_);
> + glTexImage2D(GL_TEXTURE_2D,
> + 0,
> + GL_RGBA,
> + size_.width() / 2,
> + size_.height(),
> + 0,
> + GL_RGBA,
> + GL_UNSIGNED_BYTE,
> + yuvData_);
> + shaderProgram_.setUniformValue(textureUniformY_, 0);
> +
> + /*
> + * The shader needs the step between two texture pixels in the
> + * horizontal direction, expressed in texture coordinate units
> + * ([0, 1]). There are exactly width - 1 steps between the
> + * leftmost and rightmost texels.
> + */
> + shaderProgram_.setUniformValue(textureUniformStepX_,
> + 1.0f / (size_.width() / 2 - 1));
> + break;
> +
> default:
> break;
> };
> diff --git a/src/qcam/viewfinder_gl.h b/src/qcam/viewfinder_gl.h
> index 53424dc10bc5..ad1e195e45c7 100644
> --- a/src/qcam/viewfinder_gl.h
> +++ b/src/qcam/viewfinder_gl.h
> @@ -79,6 +79,7 @@ private:
> GLuint textureUniformU_;
> GLuint textureUniformV_;
> GLuint textureUniformY_;
> + GLuint textureUniformStepX_;
> QOpenGLTexture textureU_;
> QOpenGLTexture textureV_;
> QOpenGLTexture textureY_;
> --
> Regards,
>
> Laurent Pinchart
>
> _______________________________________________
> libcamera-devel mailing list
> libcamera-devel at lists.libcamera.org
> https://lists.libcamera.org/listinfo/libcamera-devel
--
Regards,
Niklas Söderlund
More information about the libcamera-devel
mailing list