[libcamera-devel] [PATCH v5 2/9] qcam: viewfinder_gl: Add shader to render packed RAW10 formats
Laurent Pinchart
laurent.pinchart at ideasonboard.com
Wed Jun 30 02:55:03 CEST 2021
Hi Andrey,
Thank you for the patch.
On Tue, Jun 22, 2021 at 04:46:45PM +0300, Andrey Konovalov wrote:
> The shader supports all 4 packed RAW10 variants.
> Simple bi-linear Bayer interpolation of nearest pixels is implemented.
> The 2 LS bits of the 10-bit colour values are dropped as the RGBA
> format we convert into has only 8 bits per colour.
>
> The texture coordinates passed to the fragment shader are ajusted
s/ajusted/adjusted
> to point to the nearest pixel in the image. This prevents artifacts
> when the image is scaled from the frame resolution to the window size.
>
> Signed-off-by: Andrey Konovalov <andrey.konovalov at linaro.org>
> ---
> src/qcam/assets/shader/bayer_1x_packed.frag | 185 ++++++++++++++++++++
> src/qcam/assets/shader/shaders.qrc | 1 +
> src/qcam/viewfinder_gl.cpp | 78 ++++++++-
> src/qcam/viewfinder_gl.h | 9 +
> 4 files changed, 271 insertions(+), 2 deletions(-)
> create mode 100644 src/qcam/assets/shader/bayer_1x_packed.frag
>
> diff --git a/src/qcam/assets/shader/bayer_1x_packed.frag b/src/qcam/assets/shader/bayer_1x_packed.frag
> new file mode 100644
> index 00000000..fac8c3b8
> --- /dev/null
> +++ b/src/qcam/assets/shader/bayer_1x_packed.frag
> @@ -0,0 +1,185 @@
> +/* SPDX-License-Identifier: BSD-2-Clause */
> +/*
> + * Based on the code from http://jgt.akpeters.com/papers/McGuire08/
> + *
> + * Efficient, High-Quality Bayer Demosaic Filtering on GPUs
> + *
> + * Morgan McGuire
> + *
> + * This paper appears in issue Volume 13, Number 4.
> + * ---------------------------------------------------------
> + * Copyright (c) 2008, Morgan McGuire. All rights reserved.
> + *
> + *
> + * Modified by Linaro Ltd for 10/12-bit packed vs 8-bit raw Bayer format,
> + * and for simpler demosaic algorithm.
> + * Copyright (C) 2020, Linaro
> + *
> + * bayer_1x_packed.frag - Fragment shader code for raw Bayer 10-bit and 12-bit
> + * packed formats
> + */
> +
> +#ifdef GL_ES
> +precision mediump float;
> +#endif
> +
> +varying vec2 textureOut;
> +
> +/* the texture size in pixels */
> +uniform vec2 tex_size;
> +uniform vec2 tex_step;
> +uniform vec2 tex_bayer_first_red;
> +
> +uniform sampler2D tex_y;
> +
> +void main(void)
> +{
> + vec3 rgb;
> +
> + /*
> + * center_bytes holds the coordinates of the MS byte of the pixel
> + * being sampled on the [0, stride-1/height-1] range.
> + * center_pixel holds the coordinates of the pixel being sampled
> + * on the [0, width/height-1] range.
> + */
> + vec2 center_bytes;
> + vec2 center_pixel;
> +
> + /*
> + * x- and y-positions of the adjacent pixels on the [0, 1] range.
> + */
> + vec2 xcoords;
> + vec2 ycoords;
> +
> + /*
> + * The coordinates passed to the shader in textureOut may point
> + * to a place in between the pixels if the texture format doesn't
> + * match the image format. In particulr, MIPI packed raw Bayer
s/particulr/particular/
> + * formats don't have a matching texture format.
> + * In this case align the coordinates to the left nearest pixel
> + * by hand.
> + */
> + center_pixel = floor(textureOut * tex_size);
> + center_bytes.y = center_pixel.y;
> +
> + /*
> + * Add a small number (a few mantissa's LSBs) to avoid float
> + * representation issues. Maybe paranoic.
> + */
> + center_bytes.x = BPP_X * center_pixel.x + 0.02;
> +
> + const float threshold_l = 0.127 /* fract(BPP_X) * 0.5 + 0.02 */;
> + const float threshold_h = 0.625 /* 1.0 - fract(BPP_X) * 1.5 */;
> +
> + float fract_x = fract(center_bytes.x);
> +
> + /*
> + * The below floor() call ensures that center_bytes.x points
> + * at one of the bytes representing the 8 higher bits of
> + * the pixel value, not at the byte containing the LS bits
> + * of the group of the pixels.
> + */
> + center_bytes.x = floor(center_bytes.x);
> + center_bytes *= tex_step;
> +
> + xcoords = center_bytes.x + vec2(-tex_step.x, tex_step.x);
> + ycoords = center_bytes.y + vec2(-tex_step.y, tex_step.y);
> +
> + /*
> + * If xcoords[0] points at the byte containing the LS bits
> + * of the previous group of the pixels, move xcoords[0] one
> + * byte back.
> + */
> + xcoords[0] += (fract_x < threshold_l) ? -tex_step.x : 0.0;
> +
> + /*
> + * If xcoords[1] points at the byte containing the LS bits
> + * of the current group of the pixels, move xcoords[1] one
> + * byte forward.
> + */
> + xcoords[1] += (fract_x > threshold_h) ? tex_step.x : 0.0;
> +
> + vec2 alternate = mod(center_pixel.xy + tex_bayer_first_red, 2.0);
> + bool even_col = alternate.x < 1.0;
> + bool even_row = alternate.y < 1.0;
> +
> + /*
> + * We need to sample the central pixel and the ones with offset
> + * of -1 to +1 pixel in both X and Y directions. Let's name these
> + * pixels as below, where C is the central pixel:
> + *
> + * +----+----+----+----+
> + * | \ x| | | |
> + * |y \ | -1 | 0 | +1 |
There's an extra white space at the end of this line.
Those are minor issues, I'll fix them when applying the patch.
Reviewed-by: Laurent Pinchart <laurent.pinchart at ideasonboard.com>
> + * +----+----+----+----+
> + * | +1 | D2 | A1 | D3 |
> + * +----+----+----+----+
> + * | 0 | B0 | C | B1 |
> + * +----+----+----+----+
> + * | -1 | D0 | A0 | D1 |
> + * +----+----+----+----+
> + *
> + * In the below equations (0,-1).r means "r component of the texel
> + * shifted by -tex_step.y from the center_bytes one" etc.
> + *
> + * In the even row / even column (EE) case the colour values are:
> + * R = C = (0,0).r,
> + * G = (A0 + A1 + B0 + B1) / 4.0 =
> + * ( (0,-1).r + (0,1).r + (-1,0).r + (1,0).r ) / 4.0,
> + * B = (D0 + D1 + D2 + D3) / 4.0 =
> + * ( (-1,-1).r + (1,-1).r + (-1,1).r + (1,1).r ) / 4.0
> + *
> + * For even row / odd column (EO):
> + * R = (B0 + B1) / 2.0 = ( (-1,0).r + (1,0).r ) / 2.0,
> + * G = C = (0,0).r,
> + * B = (A0 + A1) / 2.0 = ( (0,-1).r + (0,1).r ) / 2.0
> + *
> + * For odd row / even column (OE):
> + * R = (A0 + A1) / 2.0 = ( (0,-1).r + (0,1).r ) / 2.0,
> + * G = C = (0,0).r,
> + * B = (B0 + B1) / 2.0 = ( (-1,0).r + (1,0).r ) / 2.0
> + *
> + * For odd row / odd column (OO):
> + * R = (D0 + D1 + D2 + D3) / 4.0 =
> + * ( (-1,-1).r + (1,-1).r + (-1,1).r + (1,1).r ) / 4.0,
> + * G = (A0 + A1 + B0 + B1) / 4.0 =
> + * ( (0,-1).r + (0,1).r + (-1,0).r + (1,0).r ) / 4.0,
> + * B = C = (0,0).r
> + */
> +
> + /*
> + * Fetch the values and precalculate the terms:
> + * patterns.x = (A0 + A1) / 2.0
> + * patterns.y = (B0 + B1) / 2.0
> + * patterns.z = (A0 + A1 + B0 + B1) / 4.0
> + * patterns.w = (D0 + D1 + D2 + D3) / 4.0
> + */
> + #define fetch(x, y) texture2D(tex_y, vec2(x, y)).r
> +
> + float C = texture2D(tex_y, center_bytes).r;
> + vec4 patterns = vec4(
> + fetch(center_bytes.x, ycoords[0]), /* A0: (0,-1) */
> + fetch(xcoords[0], center_bytes.y), /* B0: (-1,0) */
> + fetch(xcoords[0], ycoords[0]), /* D0: (-1,-1) */
> + fetch(xcoords[1], ycoords[0])); /* D1: (1,-1) */
> + vec4 temp = vec4(
> + fetch(center_bytes.x, ycoords[1]), /* A1: (0,1) */
> + fetch(xcoords[1], center_bytes.y), /* B1: (1,0) */
> + fetch(xcoords[1], ycoords[1]), /* D3: (1,1) */
> + fetch(xcoords[0], ycoords[1])); /* D2: (-1,1) */
> + patterns = (patterns + temp) * 0.5;
> + /* .x = (A0 + A1) / 2.0, .y = (B0 + B1) / 2.0 */
> + /* .z = (D0 + D3) / 2.0, .w = (D1 + D2) / 2.0 */
> + patterns.w = (patterns.z + patterns.w) * 0.5;
> + patterns.z = (patterns.x + patterns.y) * 0.5;
> +
> + rgb = even_col ?
> + (even_row ?
> + vec3(C, patterns.zw) :
> + vec3(patterns.x, C, patterns.y)) :
> + (even_row ?
> + vec3(patterns.y, C, patterns.x) :
> + vec3(patterns.wz, C));
> +
> + gl_FragColor = vec4(rgb, 1.0);
> +}
> diff --git a/src/qcam/assets/shader/shaders.qrc b/src/qcam/assets/shader/shaders.qrc
> index 8a8f9de1..d76d65c5 100644
> --- a/src/qcam/assets/shader/shaders.qrc
> +++ b/src/qcam/assets/shader/shaders.qrc
> @@ -5,6 +5,7 @@
> <file>YUV_2_planes.frag</file>
> <file>YUV_3_planes.frag</file>
> <file>YUV_packed.frag</file>
> + <file>bayer_1x_packed.frag</file>
> <file>identity.vert</file>
> </qresource>
> </RCC>
> diff --git a/src/qcam/viewfinder_gl.cpp b/src/qcam/viewfinder_gl.cpp
> index ff719418..ffbbc6c5 100644
> --- a/src/qcam/viewfinder_gl.cpp
> +++ b/src/qcam/viewfinder_gl.cpp
> @@ -36,6 +36,11 @@ static const QList<libcamera::PixelFormat> supportedFormats{
> libcamera::formats::RGBA8888,
> libcamera::formats::BGR888,
> libcamera::formats::RGB888,
> + /* Raw Bayer 10-bit packed */
> + libcamera::formats::SBGGR10_CSI2P,
> + libcamera::formats::SGBRG10_CSI2P,
> + libcamera::formats::SGRBG10_CSI2P,
> + libcamera::formats::SRGGB10_CSI2P,
> };
>
> ViewFinderGL::ViewFinderGL(QWidget *parent)
> @@ -106,6 +111,10 @@ void ViewFinderGL::render(libcamera::FrameBuffer *buffer, MappedBuffer *map)
> renderComplete(buffer_);
>
> data_ = static_cast<unsigned char *>(map->memory);
> + /*
> + * \todo Get the stride from the buffer instead of computing it naively
> + */
> + stride_ = buffer->metadata().planes[0].bytesused / size_.height();
> update();
> buffer_ = buffer;
> }
> @@ -114,6 +123,9 @@ bool ViewFinderGL::selectFormat(const libcamera::PixelFormat &format)
> {
> bool ret = true;
>
> + /* Set min/mag filters to GL_LINEAR by default. */
> + textureMinMagFilters_ = GL_LINEAR;
> +
> fragmentShaderDefines_.clear();
>
> switch (format) {
> @@ -203,6 +215,34 @@ bool ViewFinderGL::selectFormat(const libcamera::PixelFormat &format)
> fragmentShaderDefines_.append("#define RGB_PATTERN bgr");
> fragmentShaderFile_ = ":RGB.frag";
> break;
> + case libcamera::formats::SBGGR10_CSI2P:
> + firstRed_.setX(1.0);
> + firstRed_.setY(1.0);
> + fragmentShaderDefines_.append("#define BPP_X 1.25");
> + fragmentShaderFile_ = ":bayer_1x_packed.frag";
> + textureMinMagFilters_ = GL_NEAREST;
> + break;
> + case libcamera::formats::SGBRG10_CSI2P:
> + firstRed_.setX(0.0);
> + firstRed_.setY(1.0);
> + fragmentShaderDefines_.append("#define BPP_X 1.25");
> + fragmentShaderFile_ = ":bayer_1x_packed.frag";
> + textureMinMagFilters_ = GL_NEAREST;
> + break;
> + case libcamera::formats::SGRBG10_CSI2P:
> + firstRed_.setX(1.0);
> + firstRed_.setY(0.0);
> + fragmentShaderDefines_.append("#define BPP_X 1.25");
> + fragmentShaderFile_ = ":bayer_1x_packed.frag";
> + textureMinMagFilters_ = GL_NEAREST;
> + break;
> + case libcamera::formats::SRGGB10_CSI2P:
> + firstRed_.setX(0.0);
> + firstRed_.setY(0.0);
> + fragmentShaderDefines_.append("#define BPP_X 1.25");
> + fragmentShaderFile_ = ":bayer_1x_packed.frag";
> + textureMinMagFilters_ = GL_NEAREST;
> + break;
> default:
> ret = false;
> qWarning() << "[ViewFinderGL]:"
> @@ -290,6 +330,8 @@ bool ViewFinderGL::createFragmentShader()
> textureUniformU_ = shaderProgram_.uniformLocation("tex_u");
> textureUniformV_ = shaderProgram_.uniformLocation("tex_v");
> textureUniformStep_ = shaderProgram_.uniformLocation("tex_step");
> + textureUniformSize_ = shaderProgram_.uniformLocation("tex_size");
> + textureUniformBayerFirstRed_ = shaderProgram_.uniformLocation("tex_bayer_first_red");
>
> /* Create the textures. */
> for (std::unique_ptr<QOpenGLTexture> &texture : textures_) {
> @@ -306,8 +348,10 @@ bool ViewFinderGL::createFragmentShader()
> void ViewFinderGL::configureTexture(QOpenGLTexture &texture)
> {
> glBindTexture(GL_TEXTURE_2D, texture.textureId());
> - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
> - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
> + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER,
> + textureMinMagFilters_);
> + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER,
> + textureMinMagFilters_);
> glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
> glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
> }
> @@ -547,6 +591,36 @@ void ViewFinderGL::doRender()
> shaderProgram_.setUniformValue(textureUniformY_, 0);
> break;
>
> + case libcamera::formats::SBGGR10_CSI2P:
> + case libcamera::formats::SGBRG10_CSI2P:
> + case libcamera::formats::SGRBG10_CSI2P:
> + case libcamera::formats::SRGGB10_CSI2P:
> + /*
> + * Packed raw Bayer 10-bit formats are stored in GL_RED texture.
> + * The texture width is equal to the stride.
> + */
> + glActiveTexture(GL_TEXTURE0);
> + configureTexture(*textures_[0]);
> + glTexImage2D(GL_TEXTURE_2D,
> + 0,
> + GL_RED,
> + stride_,
> + size_.height(),
> + 0,
> + GL_RED,
> + GL_UNSIGNED_BYTE,
> + data_);
> + shaderProgram_.setUniformValue(textureUniformY_, 0);
> + shaderProgram_.setUniformValue(textureUniformBayerFirstRed_,
> + firstRed_);
> + shaderProgram_.setUniformValue(textureUniformSize_,
> + size_.width(), /* in pixels */
> + size_.height());
> + shaderProgram_.setUniformValue(textureUniformStep_,
> + 1.0f / (stride_ - 1),
> + 1.0f / (size_.height() - 1));
> + break;
> +
> default:
> break;
> };
> diff --git a/src/qcam/viewfinder_gl.h b/src/qcam/viewfinder_gl.h
> index 1b1faa91..508155b1 100644
> --- a/src/qcam/viewfinder_gl.h
> +++ b/src/qcam/viewfinder_gl.h
> @@ -66,6 +66,7 @@ private:
> libcamera::FrameBuffer *buffer_;
> libcamera::PixelFormat format_;
> QSize size_;
> + unsigned int stride_;
> unsigned char *data_;
>
> /* Shaders */
> @@ -81,6 +82,9 @@ private:
> /* Textures */
> std::array<std::unique_ptr<QOpenGLTexture>, 3> textures_;
>
> + /* Common texture parameters */
> + GLuint textureMinMagFilters_;
> +
> /* YUV texture parameters */
> GLuint textureUniformU_;
> GLuint textureUniformV_;
> @@ -89,6 +93,11 @@ private:
> unsigned int horzSubSample_;
> unsigned int vertSubSample_;
>
> + /* Raw Bayer texture parameters */
> + GLuint textureUniformSize_;
> + GLuint textureUniformBayerFirstRed_;
> + QPointF firstRed_;
> +
> QMutex mutex_; /* Prevent concurrent access to image_ */
> };
>
--
Regards,
Laurent Pinchart
More information about the libcamera-devel
mailing list