<div dir="ltr"><div dir="ltr">Hi Laurent,<div><br></div></div><div class="gmail_quote"><div dir="ltr" class="gmail_attr">On Sun, Sep 6, 2020 at 8:58 AM Laurent Pinchart <<a href="mailto:laurent.pinchart@ideasonboard.com">laurent.pinchart@ideasonboard.com</a>> wrote:<br></div><blockquote class="gmail_quote" style="margin:0px 0px 0px 0.8ex;border-left:1px solid rgb(204,204,204);padding-left:1ex">Hi Show,<br>
<br>
On Fri, Sep 04, 2020 at 04:43:13PM +0800, Show Liu wrote:<br>
> qcam: add OpenGL renderer<br>
> <br>
> Signed-off-by: Show Liu <<a href="mailto:show.liu@linaro.org" target="_blank">show.liu@linaro.org</a>><br>
<br>
Do I assume correctly that this was a stray file that shouldn't have<br>
been sent as part of this series ?<br></blockquote><div><br></div><div>Yes. Please skip this.</div><div>Sorry for the confusion.</div><div><br></div>Best Regards,<div>Show Liu </div><blockquote class="gmail_quote" style="margin:0px 0px 0px 0.8ex;border-left:1px solid rgb(204,204,204);padding-left:1ex">
<br>
> ---<br>
> src/qcam/meson.build | 2 +<br>
> src/qcam/renderer.cpp | 346 ++++++++++++++++++++++++++++++++++++++++++<br>
> src/qcam/renderer.h | 81 ++++++++++<br>
> 3 files changed, 429 insertions(+)<br>
> create mode 100644 src/qcam/renderer.cpp<br>
> create mode 100644 src/qcam/renderer.h<br>
> <br>
> diff --git a/src/qcam/meson.build b/src/qcam/meson.build<br>
> index e0c6f26..8c9032f 100644<br>
> --- a/src/qcam/meson.build<br>
> +++ b/src/qcam/meson.build<br>
> @@ -7,11 +7,13 @@ qcam_sources = files([<br>
> 'main.cpp',<br>
> 'main_window.cpp',<br>
> 'viewfinder.cpp',<br>
> + 'renderer.cpp'<br>
> ])<br>
> <br>
> qcam_moc_headers = files([<br>
> 'main_window.h',<br>
> 'viewfinder.h',<br>
> + 'renderer.h'<br>
> ])<br>
> <br>
> qcam_resources = files([<br>
> diff --git a/src/qcam/renderer.cpp b/src/qcam/renderer.cpp<br>
> new file mode 100644<br>
> index 0000000..23e8fa8<br>
> --- /dev/null<br>
> +++ b/src/qcam/renderer.cpp<br>
> @@ -0,0 +1,346 @@<br>
> +/* SPDX-License-Identifier: GPL-2.0-or-later */<br>
> +/*<br>
> + * Copyright (C) 2020, Linaro<br>
> + *<br>
> + * renderer.cpp - Render YUV format frame by OpenGL shader<br>
> + */<br>
> +<br>
> +#include "renderer.h"<br>
> +<br>
> +Renderer::Renderer()<br>
> + : fbo_(nullptr),<br>
> + vbo_(QOpenGLBuffer::VertexBuffer),<br>
> + pFShader_(nullptr),<br>
> + pVShader_(nullptr),<br>
> + textureU_(QOpenGLTexture::Target2D),<br>
> + textureV_(QOpenGLTexture::Target2D),<br>
> + textureY_(QOpenGLTexture::Target2D)<br>
> +{<br>
> + /* offscreen format setup */<br>
> + setFormat(requestedFormat());<br>
> + create();<br>
> +<br>
> + /* create OpenGL context */<br>
> + if (ctx_.create()) {<br>
> + ctx_.makeCurrent(this);<br>
> + initializeOpenGLFunctions();<br>
> + } else {<br>
> + qWarning() << "[Renderer]: "<br>
> + << "OpenGL renderer is not available.";<br>
> + }<br>
> +}<br>
> +<br>
> +Renderer::~Renderer()<br>
> +{<br>
> + if (vbo_.isCreated()) {<br>
> + vbo_.release();<br>
> + vbo_.destroy();<br>
> + }<br>
> +<br>
> + if (fbo_) {<br>
> + fbo_->release();<br>
> + delete fbo_;<br>
> + }<br>
> +<br>
> + removeShader();<br>
> +<br>
> + if (textureY_.isCreated())<br>
> + textureY_.destroy();<br>
> +<br>
> + if (textureU_.isCreated())<br>
> + textureU_.destroy();<br>
> +<br>
> + if (textureV_.isCreated())<br>
> + textureV_.destroy();<br>
> +<br>
> + ctx_.doneCurrent();<br>
> +}<br>
> +<br>
> +void Renderer::initializeGL()<br>
> +{<br>
> + glEnable(GL_TEXTURE_2D);<br>
> + glDisable(GL_DEPTH_TEST);<br>
> +<br>
> + static const GLfloat vertices[]{<br>
> + -1.0f, -1.0f, -1.0f, +1.0f,<br>
> + +1.0f, +1.0f, +1.0f, -1.0f,<br>
> + 0.0f, 1.0f, 0.0f, 0.0f,<br>
> + 1.0f, 0.0f, 1.0f, 1.0f<br>
> + };<br>
> +<br>
> + vbo_.create();<br>
> + vbo_.bind();<br>
> + vbo_.allocate(vertices, sizeof(vertices));<br>
> +<br>
> + glClearColor(1.0f, 1.0f, 1.0f, 0.0f);<br>
> +}<br>
> +<br>
> +bool Renderer::selectShader(const libcamera::PixelFormat &format)<br>
> +{<br>
> + bool ret = true;<br>
> + switch (format) {<br>
> + case libcamera::formats::NV12:<br>
> + horzSubSample_ = 2;<br>
> + vertSubSample_ = 2;<br>
> + vsrc_ = ":NV_vertex_shader.glsl";<br>
> + fsrc_ = ":NV_2_planes_UV_f.glsl";<br>
> + break;<br>
> + case libcamera::formats::NV21:<br>
> + horzSubSample_ = 2;<br>
> + vertSubSample_ = 2;<br>
> + vsrc_ = ":NV_vertex_shader.glsl";<br>
> + fsrc_ = ":NV_2_planes_VU_f.glsl";<br>
> + break;<br>
> + case libcamera::formats::NV16:<br>
> + horzSubSample_ = 2;<br>
> + vertSubSample_ = 1;<br>
> + vsrc_ = ":NV_vertex_shader.glsl";<br>
> + fsrc_ = ":NV_2_planes_UV_f.glsl";<br>
> + break;<br>
> + case libcamera::formats::NV61:<br>
> + horzSubSample_ = 2;<br>
> + vertSubSample_ = 1;<br>
> + vsrc_ = ":NV_vertex_shader.glsl";<br>
> + fsrc_ = ":NV_2_planes_VU_f.glsl";<br>
> + break;<br>
> + case libcamera::formats::NV24:<br>
> + horzSubSample_ = 1;<br>
> + vertSubSample_ = 1;<br>
> + vsrc_ = ":NV_vertex_shader.glsl";<br>
> + fsrc_ = ":NV_2_planes_UV_f.glsl";<br>
> + break;<br>
> + case libcamera::formats::NV42:<br>
> + horzSubSample_ = 1;<br>
> + vertSubSample_ = 1;<br>
> + vsrc_ = ":NV_vertex_shader.glsl";<br>
> + fsrc_ = ":NV_2_planes_VU_f.glsl";<br>
> + break;<br>
> + case libcamera::formats::YUV420:<br>
> + horzSubSample_ = 2;<br>
> + vertSubSample_ = 2;<br>
> + vsrc_ = ":NV_vertex_shader.glsl";<br>
> + fsrc_ = ":NV_3_planes_UV_f.glsl";<br>
> + break;<br>
> + default:<br>
> + ret = false;<br>
> + qWarning() << "[Renderer]: "<br>
> + << "format not support yet.";<br>
> + break;<br>
> + };<br>
> +<br>
> + return ret;<br>
> +}<br>
> +<br>
> +void Renderer::removeShader()<br>
> +{<br>
> + if (shaderProgram_.isLinked()) {<br>
> + shaderProgram_.release();<br>
> + shaderProgram_.removeAllShaders();<br>
> + }<br>
> +<br>
> + if (pFShader_)<br>
> + delete pFShader_;<br>
> +<br>
> + if (pVShader_)<br>
> + delete pVShader_;<br>
> +}<br>
> +<br>
> +bool Renderer::createShader()<br>
> +{<br>
> + bool bCompile;<br>
> +<br>
> + /* Create Vertex Shader */<br>
> + pVShader_ = new QOpenGLShader(QOpenGLShader::Vertex, this);<br>
> +<br>
> + bCompile = pVShader_->compileSourceFile(vsrc_);<br>
> + if (!bCompile) {<br>
> + qWarning() << "[Renderer]: " << pVShader_->log();<br>
> + return bCompile;<br>
> + }<br>
> +<br>
> + shaderProgram_.addShader(pVShader_);<br>
> +<br>
> + /* Create Fragment Shader */<br>
> + pFShader_ = new QOpenGLShader(QOpenGLShader::Fragment, this);<br>
> +<br>
> + bCompile = pFShader_->compileSourceFile(fsrc_);<br>
> + if (!bCompile) {<br>
> + qWarning() << "[Renderer]: " << pFShader_->log();<br>
> + return bCompile;<br>
> + }<br>
> +<br>
> + shaderProgram_.addShader(pFShader_);<br>
> +<br>
> + // Link shader pipeline<br>
> + if (!shaderProgram_.link()) {<br>
> + qWarning() << "[Renderer]: " << shaderProgram_.log();<br>
> + return false;<br>
> + }<br>
> +<br>
> + // Bind shader pipeline for use<br>
> + if (!shaderProgram_.bind()) {<br>
> + qWarning() << "[Renderer]: " << shaderProgram_.log();<br>
> + return false;<br>
> + }<br>
> + return true;<br>
> +}<br>
> +<br>
> +bool Renderer::configure(const libcamera::PixelFormat &format, const QSize &size)<br>
> +{<br>
> + bool ret = true;<br>
> +<br>
> + if (selectShader(format)) {<br>
> + ret = createShader();<br>
> + if (!ret)<br>
> + return ret;<br>
> +<br>
> + shaderProgram_.enableAttributeArray(ATTRIB_VERTEX);<br>
> + shaderProgram_.enableAttributeArray(ATTRIB_TEXTURE);<br>
> +<br>
> + shaderProgram_.setAttributeBuffer(ATTRIB_VERTEX,<br>
> + GL_FLOAT,<br>
> + 0,<br>
> + 2,<br>
> + 2 * sizeof(GLfloat));<br>
> + shaderProgram_.setAttributeBuffer(ATTRIB_TEXTURE,<br>
> + GL_FLOAT,<br>
> + 8 * sizeof(GLfloat),<br>
> + 2,<br>
> + 2 * sizeof(GLfloat));<br>
> +<br>
> + textureUniformY_ = shaderProgram_.uniformLocation("tex_y");<br>
> + textureUniformU_ = shaderProgram_.uniformLocation("tex_u");<br>
> + textureUniformV_ = shaderProgram_.uniformLocation("tex_v");<br>
> +<br>
> + if (!textureY_.isCreated())<br>
> + textureY_.create();<br>
> +<br>
> + if (!textureU_.isCreated())<br>
> + textureU_.create();<br>
> +<br>
> + if (!textureV_.isCreated())<br>
> + textureV_.create();<br>
> +<br>
> + id_y_ = textureY_.textureId();<br>
> + id_u_ = textureU_.textureId();<br>
> + id_v_ = textureV_.textureId();<br>
> +<br>
> + fbo_ = new QOpenGLFramebufferObject(size.width(),<br>
> + size.height(),<br>
> + GL_TEXTURE_2D);<br>
> + fbo_->bind();<br>
> + glViewport(0, 0, size.width(), size.height());<br>
> +<br>
> + format_ = format;<br>
> + size_ = size;<br>
> + } else {<br>
> + ret = false;<br>
> + }<br>
> + return ret;<br>
> +}<br>
> +<br>
> +void Renderer::configureTexture(unsigned int id)<br>
> +{<br>
> + glBindTexture(GL_TEXTURE_2D, id);<br>
> + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);<br>
> + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);<br>
> + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);<br>
> + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);<br>
> +}<br>
> +<br>
> +void Renderer::render(unsigned char *buffer)<br>
> +{<br>
> + QMutexLocker locker(&mutex_);<br>
> +<br>
> + glClearColor(0.0, 0.0, 0.0, 1.0);<br>
> + glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);<br>
> +<br>
> + switch (format_) {<br>
> + case libcamera::formats::NV12:<br>
> + case libcamera::formats::NV21:<br>
> + case libcamera::formats::NV16:<br>
> + case libcamera::formats::NV61:<br>
> + case libcamera::formats::NV24:<br>
> + case libcamera::formats::NV42:<br>
> + /* activate texture 0 */<br>
> + glActiveTexture(GL_TEXTURE0);<br>
> + configureTexture(id_y_);<br>
> + glTexImage2D(GL_TEXTURE_2D,<br>
> + 0,<br>
> + GL_RED,<br>
> + size_.width(),<br>
> + size_.height(),<br>
> + 0,<br>
> + GL_RED,<br>
> + GL_UNSIGNED_BYTE,<br>
> + buffer);<br>
> + glUniform1i(textureUniformY_, 0);<br>
> +<br>
> + /* activate texture 1 */<br>
> + glActiveTexture(GL_TEXTURE1);<br>
> + configureTexture(id_u_);<br>
> + glTexImage2D(GL_TEXTURE_2D,<br>
> + 0,<br>
> + GL_RG,<br>
> + size_.width() / horzSubSample_,<br>
> + size_.height() / vertSubSample_,<br>
> + 0,<br>
> + GL_RG,<br>
> + GL_UNSIGNED_BYTE,<br>
> + buffer + size_.width() * size_.height());<br>
> + glUniform1i(textureUniformU_, 1);<br>
> + break;<br>
> + case libcamera::formats::YUV420:<br>
> + /* activate texture 0 */<br>
> + glActiveTexture(GL_TEXTURE0);<br>
> + configureTexture(id_y_);<br>
> + glTexImage2D(GL_TEXTURE_2D,<br>
> + 0,<br>
> + GL_RED,<br>
> + size_.width(),<br>
> + size_.height(),<br>
> + 0,<br>
> + GL_RED,<br>
> + GL_UNSIGNED_BYTE,<br>
> + buffer);<br>
> + glUniform1i(textureUniformY_, 0);<br>
> +<br>
> + /* activate texture 1 */<br>
> + glActiveTexture(GL_TEXTURE1);<br>
> + configureTexture(id_u_);<br>
> + glTexImage2D(GL_TEXTURE_2D,<br>
> + 0,<br>
> + GL_RG,<br>
> + size_.width() / horzSubSample_,<br>
> + size_.height() / vertSubSample_,<br>
> + 0,<br>
> + GL_RG,<br>
> + GL_UNSIGNED_BYTE,<br>
> + buffer + size_.width() * size_.height());<br>
> + glUniform1i(textureUniformU_, 1);<br>
> +<br>
> + /* activate texture 2 */<br>
> + glActiveTexture(GL_TEXTURE2);<br>
> + configureTexture(id_v_);<br>
> + glTexImage2D(GL_TEXTURE_2D,<br>
> + 0,<br>
> + GL_RG,<br>
> + size_.width() / horzSubSample_,<br>
> + size_.height() / vertSubSample_,<br>
> + 0, GL_RG,<br>
> + GL_UNSIGNED_BYTE,<br>
> + buffer + size_.width() * size_.height() * 5 / 4);<br>
> + glUniform1i(textureUniformV_, 2);<br>
> + break;<br>
> + default:<br>
> + break;<br>
> + };<br>
> +<br>
> + glDrawArrays(GL_TRIANGLE_FAN, 0, 4);<br>
> +}<br>
> +<br>
> +QImage Renderer::toImage()<br>
> +{<br>
> + QMutexLocker locker(&mutex_);<br>
> + return (fbo_->toImage(true));<br>
> +}<br>
> diff --git a/src/qcam/renderer.h b/src/qcam/renderer.h<br>
> new file mode 100644<br>
> index 0000000..1ea0c48<br>
> --- /dev/null<br>
> +++ b/src/qcam/renderer.h<br>
> @@ -0,0 +1,81 @@<br>
> +/* SPDX-License-Identifier: GPL-2.0-or-later */<br>
> +/*<br>
> + * Copyright (C) 2020, Linaro<br>
> + *<br>
> + * renderer.h - Render YUV format frame by OpenGL shader<br>
> + */<br>
> +#ifndef __QCAM_RENDERER_H__<br>
> +#define __QCAM_RENDERER_H__<br>
> +<br>
> +#include <QImage><br>
> +#include <QMutex><br>
> +#include <QOffscreenSurface><br>
> +#include <QOpenGLBuffer><br>
> +#include <QOpenGLContext><br>
> +#include <QOpenGLFramebufferObject><br>
> +#include <QOpenGLFunctions><br>
> +#include <QOpenGLShader><br>
> +#include <QOpenGLShaderProgram><br>
> +#include <QOpenGLTexture><br>
> +#include <QSize><br>
> +#include <QSurfaceFormat><br>
> +<br>
> +#include <libcamera/formats.h><br>
> +<br>
> +#define ATTRIB_VERTEX 0<br>
> +#define ATTRIB_TEXTURE 1<br>
> +<br>
> +class Renderer : public QOffscreenSurface, protected QOpenGLFunctions<br>
> +{<br>
> + Q_OBJECT<br>
> +<br>
> +public:<br>
> + Renderer();<br>
> + ~Renderer();<br>
> +<br>
> + void initializeGL();<br>
> + bool configure(const libcamera::PixelFormat &format, const QSize &size);<br>
> + void render(unsigned char *buffer);<br>
> + QImage toImage();<br>
> +<br>
> +private:<br>
> + bool createShader();<br>
> + void configureTexture(unsigned int id);<br>
> + void removeShader();<br>
> + bool selectShader(const libcamera::PixelFormat &format);<br>
> +<br>
> + /* OpenGL renderer components */<br>
> + QOpenGLContext ctx_;<br>
> + QOpenGLFramebufferObject *fbo_;<br>
> + QOpenGLBuffer vbo_;<br>
> + QOpenGLShader *pFShader_;<br>
> + QOpenGLShader *pVShader_;<br>
> + QOpenGLShaderProgram shaderProgram_;<br>
> + QSurfaceFormat surfaceFormat_;<br>
> +<br>
> + /* Fragment and Vertex shader file */<br>
> + QString fsrc_;<br>
> + QString vsrc_;<br>
> +<br>
> + /* YUV frame size and format */<br>
> + libcamera::PixelFormat format_;<br>
> + QSize size_;<br>
> +<br>
> + /* YUV texture planars and parameters*/<br>
> + GLuint id_u_;<br>
> + GLuint id_v_;<br>
> + GLuint id_y_;<br>
> + GLuint textureUniformU_;<br>
> + GLuint textureUniformV_;<br>
> + GLuint textureUniformY_;<br>
> + QOpenGLTexture textureU_;<br>
> + QOpenGLTexture textureV_;<br>
> + QOpenGLTexture textureY_;<br>
> + unsigned int horzSubSample_;<br>
> + unsigned int vertSubSample_;<br>
> +<br>
> + QImage image_;<br>
> + QMutex mutex_; /* Prevent concurrent access to image_ */<br>
> +};<br>
> +<br>
> +#endif /* __QCAM_RENDERER_H__ */<br>
<br>
-- <br>
Regards,<br>
<br>
Laurent Pinchart<br>
</blockquote></div></div>