[libcamera-devel] [PATCH v2 5/5] qcam: viewfinder_gl: Add support for RAW8 Bayer formats
Laurent Pinchart
laurent.pinchart at ideasonboard.com
Thu Jun 10 00:17:12 CEST 2021
Hi Andrey,
Thank you for the patch.
I would also like to thank Morgan for his code.
On Thu, May 27, 2021 at 01:55:11PM +0300, Andrey Konovalov wrote:
> All the four Bayer orders are supported.
>
> The texture coordinates passed to the fragment shader are ajusted
> to point to the nearest pixel in the image. This prevents artifacts
> when the image is scaled from the frame resolution to the window size.
>
> Signed-off-by: Andrey Konovalov <andrey.konovalov at linaro.org>
> ---
> src/qcam/assets/shader/bayer_8.frag | 136 ++++++++++++++++++++++++++++
> src/qcam/assets/shader/bayer_8.vert | 72 +++++++++++++++
> src/qcam/assets/shader/shaders.qrc | 1 +
> src/qcam/viewfinder_gl.cpp | 37 +++++++-
> 4 files changed, 244 insertions(+), 2 deletions(-)
> create mode 100644 src/qcam/assets/shader/bayer_8.frag
> create mode 100644 src/qcam/assets/shader/bayer_8.vert
>
> diff --git a/src/qcam/assets/shader/bayer_8.frag b/src/qcam/assets/shader/bayer_8.frag
> new file mode 100644
> index 00000000..d93ef1da
> --- /dev/null
> +++ b/src/qcam/assets/shader/bayer_8.frag
> @@ -0,0 +1,136 @@
> +/* SPDX-License-Identifier: BSD-2-Clause */
> +/*
> +From http://jgt.akpeters.com/papers/McGuire08/
> +
> +Efficient, High-Quality Bayer Demosaic Filtering on GPUs
> +
> +Morgan McGuire
> +
> +This paper appears in issue Volume 13, Number 4.
> +---------------------------------------------------------
> +Copyright (c) 2008, Morgan McGuire. All rights reserved.
> +
> +
> +Modified by Linaro Ltd to integrate it into libcamera, and to
> +fix the artifacts due to pixel coordinates interpolation.
> +Copyright (C) 2021, Linaro
Would it make sense to import the original version as-is in a patch, and
then add the changes on top ? It would be easier to review them.
> +*/
> +
> +//Pixel Shader
We try to standardize on C-style comments.
> +
> +varying vec2 textureOut;
> +
> +/* The texture size: tex_size.xy is in bytes, tex_size.zw is in pixels */
> +uniform vec4 tex_size;
> +uniform vec2 tex_step;
> +
> +/** Pixel position of the first red pixel in the */
> +/** Bayer pattern. [{0,1}, {0, 1}]*/
> +uniform vec2 tex_bayer_first_red;
> +
> +/** Monochrome RGBA or GL_LUMINANCE Bayer encoded texture.*/
> +uniform sampler2D tex_raw;
> +
> +void main(void) {
> + #define fetch(x, y) texture2D(tex_raw, vec2(x, y)).r
> +
> + /** .xy = Pixel being sampled in the fragment shader on the range [0, 1]
> + .zw = ...on the range [0, sourceSize], offset by firstRed */
For multiblock comments, our coding style is
/*
* .xy = Pixel being sampled in the fragment shader on the range [0, 1]
* .zw = ...on the range [0, sourceSize], offset by firstRed
*/
This being said, if we want to minimize changes with the original, I
would be OK.
> + vec4 center;
> +
> + /** center.x + (-2/w, -1/w, 1/w, 2/w); These are the x-positions */
> + /** of the adjacent pixels.*/
> + vec4 xCoord;
> +
> + /** center.y + (-2/h, -1/h, 1/h, 2/h); These are the y-positions */
> + /** of the adjacent pixels.*/
> + vec4 yCoord;
Doesn't computing the center, xCoord and yCoord values here defeat the
point of computing them in the vertex shader, where the interpolator
hardware can be used for the computation ?
> +
> + /* Align the center coordinates to the nearest pixel */
> + center.zw = floor(textureOut * tex_size.zw);
> + center.xy = center.zw * tex_step;
> + center.zw += tex_bayer_first_red;
> +
> + xCoord = center.x + vec4(-2.0 * tex_step.x,
> + -tex_step.x, tex_step.x, 2.0 * tex_step.x);
> + yCoord = center.y + vec4(-2.0 * tex_step.y,
> + -tex_step.y, tex_step.y, 2.0 * tex_step.y);
> +
> + float C = texture2D(tex_raw, center.xy).r; // ( 0, 0)
> + const vec4 kC = vec4( 4.0, 6.0, 5.0, 5.0) / 8.0;
> +
> + // Determine which of four types of pixels we are on.
> + vec2 alternate = mod(floor(center.zw), 2.0);
> +
> + vec4 Dvec = vec4(
> + fetch(xCoord[1], yCoord[1]), // (-1,-1)
> + fetch(xCoord[1], yCoord[2]), // (-1, 1)
> + fetch(xCoord[2], yCoord[1]), // ( 1,-1)
> + fetch(xCoord[2], yCoord[2])); // ( 1, 1)
> +
> + vec4 PATTERN = (kC.xyz * C).xyzz;
> +
> + // Can also be a dot product with (1,1,1,1) on hardware where that is
> + // specially optimized.
> + // Equivalent to: D = Dvec[0] + Dvec[1] + Dvec[2] + Dvec[3];
> + Dvec.xy += Dvec.zw;
> + Dvec.x += Dvec.y;
> +
> + vec4 value = vec4(
> + fetch(center.x, yCoord[0]), // ( 0,-2)
> + fetch(center.x, yCoord[1]), // ( 0,-1)
> + fetch(xCoord[0], center.y), // (-2, 0)
> + fetch(xCoord[1], center.y)); // (-1, 0)
> +
> + vec4 temp = vec4(
> + fetch(center.x, yCoord[3]), // ( 0, 2)
> + fetch(center.x, yCoord[2]), // ( 0, 1)
> + fetch(xCoord[3], center.y), // ( 2, 0)
> + fetch(xCoord[2], center.y)); // ( 1, 0)
> +
> + // Even the simplest compilers should be able to constant-fold these to
> + // avoid the division.
> + // Note that on scalar processors these constants force computation of some
> + // identical products twice.
> + const vec4 kA = vec4(-1.0, -1.5, 0.5, -1.0) / 8.0;
> + const vec4 kB = vec4( 2.0, 0.0, 0.0, 4.0) / 8.0;
> + const vec4 kD = vec4( 0.0, 2.0, -1.0, -1.0) / 8.0;
> +
> + // Conserve constant registers and take advantage of free swizzle on load
> + #define kE (kA.xywz)
> + #define kF (kB.xywz)
> +
> + value += temp;
> +
> + // There are five filter patterns (identity, cross, checker,
> + // theta, phi). Precompute the terms from all of them and then
> + // use swizzles to assign to color channels.
> + //
> + // Channel Matches
> + // x cross (e.g., EE G)
> + // y checker (e.g., EE B)
> + // z theta (e.g., EO R)
> + // w phi (e.g., EO R)
> + #define A (value[0])
> + #define B (value[1])
> + #define D (Dvec.x)
> + #define E (value[2])
> + #define F (value[3])
> +
> + // Avoid zero elements. On a scalar processor this saves two MADDs
> + // and it has no effect on a vector processor.
> + PATTERN.yzw += (kD.yz * D).xyy;
> +
> + PATTERN += (kA.xyz * A).xyzx + (kE.xyw * E).xyxz;
> + PATTERN.xw += kB.xw * B;
> + PATTERN.xz += kF.xz * F;
> +
> + vec3 rgb = (alternate.y == 0.0) ?
> + ((alternate.x == 0.0) ?
> + vec3(C, PATTERN.xy) :
> + vec3(PATTERN.z, C, PATTERN.w)) :
> + ((alternate.x == 0.0) ?
> + vec3(PATTERN.w, C, PATTERN.z) :
> + vec3(PATTERN.yx, C));
> + gl_FragColor = vec4(rgb, 1.0);
> +}
> diff --git a/src/qcam/assets/shader/bayer_8.vert b/src/qcam/assets/shader/bayer_8.vert
> new file mode 100644
> index 00000000..26525db7
> --- /dev/null
> +++ b/src/qcam/assets/shader/bayer_8.vert
> @@ -0,0 +1,72 @@
This needs and SPDX license identifier:
/* SPDX-License-Identifier: BSD-2-Clause */
> +/*
> +From http://jgt.akpeters.com/papers/McGuire08/
> +
> +Efficient, High-Quality Bayer Demosaic Filtering on GPUs
> +
> +Morgan McGuire
> +
> +This paper appears in issue Volume 13, Number 4.
> +---------------------------------------------------------
> +Copyright (c) 2008, Morgan McGuire. All rights reserved.
> +
> +Redistribution and use in source and binary forms, with or without
> +modification, are permitted provided that the following conditions are
> +met:
> +
> + * Redistributions of source code must retain the above copyright
> + notice, this list of conditions and the following disclaimer.
> +
> + * Redistributions in binary form must reproduce the above
> + copyright notice, this list of conditions and the following
> + disclaimer in the documentation and/or other materials provided
> + with the distribution.
> +
> +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
> +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
> +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
> +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
> +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
> +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
> +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
> +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
> +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
> +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
> +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
The whole text from "Redistribution ..." to here is conveyed by the SPDX
tag, so you can drop it.
> +*/
> +
> +//Vertex Shader
> +
> +attribute vec4 vertexIn;
> +attribute vec2 textureIn;
> +
> +/* The texture size: tex_size.xy is in bytes, tex_size.zw is in pixels */
> +uniform vec4 tex_size;
> +uniform vec2 tex_step;
> +
> +/** Pixel position of the first red pixel in the */
> +/** Bayer pattern. [{0,1}, {0, 1}]*/
> +uniform vec2 tex_bayer_first_red;
> +
> +/** .xy = Pixel being sampled in the fragment shader on the range [0, 1]
> + .zw = ...on the range [0, sourceSize], offset by firstRed */
> +varying vec4 center;
> +
> +/** center.x + (-2/w, -1/w, 1/w, 2/w); These are the x-positions */
> +/** of the adjacent pixels.*/
> +varying vec4 xCoord;
> +
> +/** center.y + (-2/h, -1/h, 1/h, 2/h); These are the y-positions */
> +/** of the adjacent pixels.*/
> +varying vec4 yCoord;
> +
> +void main(void) {
> + center.xy = textureIn;
> + center.zw = textureIn * tex_size.zw + tex_bayer_first_red;
> +
> + xCoord = center.x + vec4(-2.0 * tex_step.x,
> + -tex_step.x, tex_step.x, 2.0 * tex_step.x);
> + yCoord = center.y + vec4(-2.0 * tex_step.y,
> + -tex_step.y, tex_step.y, 2.0 * tex_step.y);
> +
> + gl_Position = vertexIn;
> +}
> diff --git a/src/qcam/assets/shader/shaders.qrc b/src/qcam/assets/shader/shaders.qrc
> index d76d65c5..79f44a30 100644
> --- a/src/qcam/assets/shader/shaders.qrc
> +++ b/src/qcam/assets/shader/shaders.qrc
> @@ -5,6 +5,7 @@
> <file>YUV_2_planes.frag</file>
> <file>YUV_3_planes.frag</file>
> <file>YUV_packed.frag</file>
> + <file>bayer_8.frag</file>
Could you please move this after bayer_1x_packed.frag to keep them
alphabetically sorted ?
You're missing bayer_8.vert.
> <file>bayer_1x_packed.frag</file>
> <file>identity.vert</file>
> </qresource>
> diff --git a/src/qcam/viewfinder_gl.cpp b/src/qcam/viewfinder_gl.cpp
> index 13ab31aa..ef85e93d 100644
> --- a/src/qcam/viewfinder_gl.cpp
> +++ b/src/qcam/viewfinder_gl.cpp
> @@ -36,6 +36,11 @@ static const QList<libcamera::PixelFormat> supportedFormats{
> libcamera::formats::RGBA8888,
> libcamera::formats::BGR888,
> libcamera::formats::RGB888,
> + /* Raw Bayer 8-bit */
> + libcamera::formats::SBGGR8,
> + libcamera::formats::SGBRG8,
> + libcamera::formats::SGRBG8,
> + libcamera::formats::SRGGB8,
> /* Raw Bayer 10-bit packed */
> libcamera::formats::SBGGR10_CSI2P,
> libcamera::formats::SGBRG10_CSI2P,
> @@ -220,6 +225,30 @@ bool ViewFinderGL::selectFormat(const libcamera::PixelFormat &format)
> fragmentShaderDefines_.append("#define RGB_PATTERN bgr");
> fragmentShaderFile_ = ":RGB.frag";
> break;
> + case libcamera::formats::SBGGR8:
> + firstRed_.setX(1.0);
> + firstRed_.setY(1.0);
> + fragmentShaderFile_ = ":bayer_8.frag";
> + textureMinMagFilters_ = GL_NEAREST;
> + break;
> + case libcamera::formats::SGBRG8:
> + firstRed_.setX(0.0);
> + firstRed_.setY(1.0);
> + fragmentShaderFile_ = ":bayer_8.frag";
> + textureMinMagFilters_ = GL_NEAREST;
> + break;
> + case libcamera::formats::SGRBG8:
> + firstRed_.setX(1.0);
> + firstRed_.setY(0.0);
> + fragmentShaderFile_ = ":bayer_8.frag";
> + textureMinMagFilters_ = GL_NEAREST;
> + break;
> + case libcamera::formats::SRGGB8:
> + firstRed_.setX(0.0);
> + firstRed_.setY(0.0);
> + fragmentShaderFile_ = ":bayer_8.frag";
> + textureMinMagFilters_ = GL_NEAREST;
> + break;
> case libcamera::formats::SBGGR10_CSI2P:
> firstRed_.setX(1.0);
> firstRed_.setY(1.0);
> @@ -624,6 +653,10 @@ void ViewFinderGL::doRender()
> shaderProgram_.setUniformValue(textureUniformY_, 0);
> break;
>
> + case libcamera::formats::SBGGR8:
> + case libcamera::formats::SGBRG8:
> + case libcamera::formats::SGRBG8:
> + case libcamera::formats::SRGGB8:
> case libcamera::formats::SBGGR10_CSI2P:
> case libcamera::formats::SGBRG10_CSI2P:
> case libcamera::formats::SGRBG10_CSI2P:
> @@ -633,8 +666,8 @@ void ViewFinderGL::doRender()
> case libcamera::formats::SGRBG12_CSI2P:
> case libcamera::formats::SRGGB12_CSI2P:
> /*
> - * Packed raw Bayer 10-bit and 12-bit formats are stored in
> - * GL_RED texture.
> + * Raw Bayer 8-bit, and packed raw Bayer 10-bit/12-bit formats
> + * are stored in GL_RED texture.
> */
> glActiveTexture(GL_TEXTURE0);
> configureTexture(*textures_[0]);
--
Regards,
Laurent Pinchart
More information about the libcamera-devel
mailing list