[libcamera-devel] [PATCH 7/7] qcam: viewfinder_gl: Add shader to render packed YUV formats
Laurent Pinchart
laurent.pinchart at ideasonboard.com
Wed Sep 16 17:56:55 CEST 2020
Hi Niklas,
On Wed, Sep 16, 2020 at 05:55:20PM +0200, Niklas Söderlund wrote:
> Hi Laurent,
>
> Thanks for your work.
>
> On 2020-09-16 17:52:54 +0300, Laurent Pinchart wrote:
> > The shader supports all 4 packed 8-bit YUV variants.
>
> OpenGL is not my strong suite so I can't really give it a proper review.
> From what I do understand this change looks good but I'm unable to give
> a R-b in good concious with my limited understanding. I do however think
> the change is neatly modularized and properly tested with good results
> so I think merging it adds value without any risk for regression or
> later problems.
>
> Acked-by: Niklas Söderlund <niklas.soderlund at ragnatech.se>
>
> Please see bellow for a small typo tho.
>
> > Signed-off-by: Laurent Pinchart <laurent.pinchart at ideasonboard.com>
> > ---
> > src/qcam/assets/shader/YUV_packed.frag | 82 ++++++++++++++++++++++++++
> > src/qcam/assets/shader/shaders.qrc | 1 +
> > src/qcam/viewfinder_gl.cpp | 56 ++++++++++++++++++
> > src/qcam/viewfinder_gl.h | 1 +
> > 4 files changed, 140 insertions(+)
> > create mode 100644 src/qcam/assets/shader/YUV_packed.frag
> >
> > diff --git a/src/qcam/assets/shader/YUV_packed.frag b/src/qcam/assets/shader/YUV_packed.frag
> > new file mode 100644
> > index 000000000000..224dfafe383e
> > --- /dev/null
> > +++ b/src/qcam/assets/shader/YUV_packed.frag
> > @@ -0,0 +1,82 @@
> > +/* SPDX-License-Identifier: LGPL-2.1-or-later */
> > +/*
> > + * Copyright (C) 2020, Laurent Pinchart <laurent.pinchart at ideasonboard.com>
> > + *
> > + * YUV_packed.frag - Fragment shader code for YUYV packed formats
> > + */
> > +
> > +#ifdef GL_ES
> > +precision mediump float;
> > +#endif
> > +
> > +varying vec2 textureOut;
> > +
> > +uniform sampler2D tex_y;
> > +uniform float tex_stepx;
> > +
> > +void main(void)
> > +{
> > + mat3 yuv2rgb_bt601_mat = mat3(
> > + vec3(1.164, 1.164, 1.164),
> > + vec3(0.000, -0.392, 2.017),
> > + vec3(1.596, -0.813, 0.000)
> > + );
> > + vec3 yuv2rgb_bt601_offset = vec3(0.063, 0.500, 0.500);
> > +
> > + /*
> > + * The sampler won't interpolate the texture correctly along the X axis,
> > + * as each RGBA pixel effectively stores two pixels. We thus need to
> > + * interpolate manually.
> > + *
> > + * In integer texture coordinates, the Y values are layed out in the
> > + * texture memory as follows:
> > + *
> > + * ...| Y U Y V | Y U Y V | Y U Y V |...
> > + * ...| R G B A | R G B A | R G B A |...
> > + * ^ ^ ^ ^ ^ ^
> > + * | | | | | |
> > + * n-1 n-0.5 n n+0.5 n+1 n+1.5
> > + *
> > + * For a texture location x in the interval [n, n+1[, sample the left
>
> s/[n, n+1[/[n, n+1]/
This was on purpose, it means an interval from n to n+1 that includes n
but doesn't include n+1. If x was equal to n+1, it would be in the
interval [n', n'+1[ where n' = n+1 :-)
> > + * and right pixels at n and n+1, and interpolate them with
> > + *
> > + * left.r * (1 - a) + left.b * a if fract(x) < 0.5
> > + * left.b * (1 - a) + right.r * a if fract(x) >= 0.5
> > + *
> > + * with a = fract(x * 2) which can also be written
> > + *
> > + * a = fract(x) * 2 if fract(x) < 0.5
> > + * a = fract(x) * 2 - 1 if fract(x) >= 0.5
> > + */
> > + vec2 pos = textureOut;
> > + float f_x = fract(pos.x / tex_stepx);
> > +
> > + vec4 left = texture2D(tex_y, vec2(pos.x - f_x * tex_stepx, pos.y));
> > + vec4 right = texture2D(tex_y, vec2(pos.x + (1.0 - f_x) * tex_stepx , pos.y));
> > +
> > +#if defined(YUV_PATTERN_UYVY)
> > + float y_left = mix(left.g, left.a, f_x * 2.0);
> > + float y_right = mix(left.a, right.g, f_x * 2.0 - 1.0);
> > + vec2 uv = mix(left.rb, right.rb, f_x);
> > +#elif defined(YUV_PATTERN_VYUY)
> > + float y_left = mix(left.g, left.a, f_x * 2.0);
> > + float y_right = mix(left.a, right.g, f_x * 2.0 - 1.0);
> > + vec2 uv = mix(left.br, right.br, f_x);
> > +#elif defined(YUV_PATTERN_YUYV)
> > + float y_left = mix(left.r, left.b, f_x * 2.0);
> > + float y_right = mix(left.b, right.r, f_x * 2.0 - 1.0);
> > + vec2 uv = mix(left.ga, right.ga, f_x);
> > +#elif defined(YUV_PATTERN_YVYU)
> > + float y_left = mix(left.r, left.b, f_x * 2.0);
> > + float y_right = mix(left.b, right.r, f_x * 2.0 - 1.0);
> > + vec2 uv = mix(left.ag, right.ag, f_x);
> > +#else
> > +#error Invalid pattern
> > +#endif
> > +
> > + float y = mix(y_left, y_right, step(0.5, f_x));
> > +
> > + vec3 rgb = yuv2rgb_bt601_mat * (vec3(y, uv) - yuv2rgb_bt601_offset);
> > +
> > + gl_FragColor = vec4(rgb, 1.0);
> > +}
> > diff --git a/src/qcam/assets/shader/shaders.qrc b/src/qcam/assets/shader/shaders.qrc
> > index 7010d8433c9b..857ed9fd5cde 100644
> > --- a/src/qcam/assets/shader/shaders.qrc
> > +++ b/src/qcam/assets/shader/shaders.qrc
> > @@ -4,5 +4,6 @@
> > <file>YUV.vert</file>
> > <file>YUV_2_planes.frag</file>
> > <file>YUV_3_planes.frag</file>
> > + <file>YUV_packed.frag</file>
> > </qresource>
> > </RCC>
> > diff --git a/src/qcam/viewfinder_gl.cpp b/src/qcam/viewfinder_gl.cpp
> > index b8a4827267c3..0b5c942658cd 100644
> > --- a/src/qcam/viewfinder_gl.cpp
> > +++ b/src/qcam/viewfinder_gl.cpp
> > @@ -14,12 +14,19 @@
> > #include <libcamera/formats.h>
> >
> > static const QList<libcamera::PixelFormat> supportedFormats{
> > + /* Packed (single plane) */
> > + libcamera::formats::UYVY,
> > + libcamera::formats::VYUY,
> > + libcamera::formats::YUYV,
> > + libcamera::formats::YVYU,
> > + /* Semi planar (two planes) */
> > libcamera::formats::NV12,
> > libcamera::formats::NV21,
> > libcamera::formats::NV16,
> > libcamera::formats::NV61,
> > libcamera::formats::NV24,
> > libcamera::formats::NV42,
> > + /* Fully planar (three planes) */
> > libcamera::formats::YUV420,
> > libcamera::formats::YVU420,
> > };
> > @@ -149,6 +156,22 @@ bool ViewFinderGL::selectFormat(const libcamera::PixelFormat &format)
> > vertSubSample_ = 2;
> > fragmentShaderFile_ = ":YUV_3_planes.frag";
> > break;
> > + case libcamera::formats::UYVY:
> > + fragmentShaderDefines_.append("#define YUV_PATTERN_UYVY");
> > + fragmentShaderFile_ = ":YUV_packed.frag";
> > + break;
> > + case libcamera::formats::VYUY:
> > + fragmentShaderDefines_.append("#define YUV_PATTERN_VYUY");
> > + fragmentShaderFile_ = ":YUV_packed.frag";
> > + break;
> > + case libcamera::formats::YUYV:
> > + fragmentShaderDefines_.append("#define YUV_PATTERN_YUYV");
> > + fragmentShaderFile_ = ":YUV_packed.frag";
> > + break;
> > + case libcamera::formats::YVYU:
> > + fragmentShaderDefines_.append("#define YUV_PATTERN_YVYU");
> > + fragmentShaderFile_ = ":YUV_packed.frag";
> > + break;
> > default:
> > ret = false;
> > qWarning() << "[ViewFinderGL]:"
> > @@ -235,6 +258,7 @@ bool ViewFinderGL::createFragmentShader()
> > textureUniformY_ = shaderProgram_.uniformLocation("tex_y");
> > textureUniformU_ = shaderProgram_.uniformLocation("tex_u");
> > textureUniformV_ = shaderProgram_.uniformLocation("tex_v");
> > + textureUniformStepX_ = shaderProgram_.uniformLocation("tex_stepx");
> >
> > if (!textureY_.isCreated())
> > textureY_.create();
> > @@ -431,6 +455,38 @@ void ViewFinderGL::doRender()
> > shaderProgram_.setUniformValue(textureUniformU_, 1);
> > break;
> >
> > + case libcamera::formats::UYVY:
> > + case libcamera::formats::VYUY:
> > + case libcamera::formats::YUYV:
> > + case libcamera::formats::YVYU:
> > + /*
> > + * Packed YUV formats are stored in a RGBA texture to match the
> > + * OpenGL texel size with the 4 bytes repeating pattern in YUV.
> > + * The texture width is thus half of the image with.
> > + */
> > + glActiveTexture(GL_TEXTURE0);
> > + configureTexture(textureY_);
> > + glTexImage2D(GL_TEXTURE_2D,
> > + 0,
> > + GL_RGBA,
> > + size_.width() / 2,
> > + size_.height(),
> > + 0,
> > + GL_RGBA,
> > + GL_UNSIGNED_BYTE,
> > + yuvData_);
> > + shaderProgram_.setUniformValue(textureUniformY_, 0);
> > +
> > + /*
> > + * The shader needs the step between two texture pixels in the
> > + * horizontal direction, expressed in texture coordinate units
> > + * ([0, 1]). There are exactly width - 1 steps between the
> > + * leftmost and rightmost texels.
> > + */
> > + shaderProgram_.setUniformValue(textureUniformStepX_,
> > + 1.0f / (size_.width() / 2 - 1));
> > + break;
> > +
> > default:
> > break;
> > };
> > diff --git a/src/qcam/viewfinder_gl.h b/src/qcam/viewfinder_gl.h
> > index 53424dc10bc5..ad1e195e45c7 100644
> > --- a/src/qcam/viewfinder_gl.h
> > +++ b/src/qcam/viewfinder_gl.h
> > @@ -79,6 +79,7 @@ private:
> > GLuint textureUniformU_;
> > GLuint textureUniformV_;
> > GLuint textureUniformY_;
> > + GLuint textureUniformStepX_;
> > QOpenGLTexture textureU_;
> > QOpenGLTexture textureV_;
> > QOpenGLTexture textureY_;
--
Regards,
Laurent Pinchart
More information about the libcamera-devel
mailing list