[libcamera-devel] [PATCH v6 3/4] qcam: add viewfinderGL class to accelerate the format convert
Show Liu
show.liu at linaro.org
Fri Sep 11 10:55:13 CEST 2020
the viewfinderGL accelerates the format conversion by
using OpenGL ES shader
Signed-off-by: Show Liu <show.liu at linaro.org>
---
meson.build | 1 +
src/qcam/meson.build | 17 +-
src/qcam/viewfinder_gl.cpp | 456 +++++++++++++++++++++++++++++++++++++
src/qcam/viewfinder_gl.h | 96 ++++++++
4 files changed, 568 insertions(+), 2 deletions(-)
create mode 100644 src/qcam/viewfinder_gl.cpp
create mode 100644 src/qcam/viewfinder_gl.h
diff --git a/meson.build b/meson.build
index 1ea35e9..c58d458 100644
--- a/meson.build
+++ b/meson.build
@@ -26,6 +26,7 @@ libcamera_version = libcamera_git_version.split('+')[0]
# Configure the build environment.
cc = meson.get_compiler('c')
+cxx = meson.get_compiler('cpp')
config_h = configuration_data()
if cc.has_header_symbol('execinfo.h', 'backtrace')
diff --git a/src/qcam/meson.build b/src/qcam/meson.build
index a4bad0a..9bb48c0 100644
--- a/src/qcam/meson.build
+++ b/src/qcam/meson.build
@@ -16,14 +16,14 @@ qcam_moc_headers = files([
qcam_resources = files([
'assets/feathericons/feathericons.qrc',
- 'assets/shader/shaders.qrc'
])
qt5 = import('qt5')
qt5_dep = dependency('qt5',
method : 'pkg-config',
modules : ['Core', 'Gui', 'Widgets'],
- required : get_option('qcam'))
+ required : get_option('qcam'),
+ version : '>=5.4')
if qt5_dep.found()
qcam_deps = [
@@ -42,6 +42,19 @@ if qt5_dep.found()
])
endif
+ if cxx.has_header_symbol('QOpenGLWidget', 'QOpenGLWidget',
+ dependencies : qt5_dep, args : '-fPIC')
+ qcam_sources += files([
+ 'viewfinder_gl.cpp',
+ ])
+ qcam_moc_headers += files([
+ 'viewfinder_gl.h',
+ ])
+ qcam_resources += files([
+ 'assets/shader/shaders.qrc'
+ ])
+ endif
+
# gcc 9 introduced a deprecated-copy warning that is triggered by Qt until
# Qt 5.13. clang 10 introduced the same warning, but detects more issues
# that are not fixed in Qt yet. Disable the warning manually in both cases.
diff --git a/src/qcam/viewfinder_gl.cpp b/src/qcam/viewfinder_gl.cpp
new file mode 100644
index 0000000..84f4866
--- /dev/null
+++ b/src/qcam/viewfinder_gl.cpp
@@ -0,0 +1,456 @@
+/* SPDX-License-Identifier: LGPL-2.1-or-later */
+/*
+ * Copyright (C) 2020, Linaro
+ *
+ * viewfinderGL.cpp - OpenGL Viewfinder for rendering by OpenGL shader
+ */
+
+#include "viewfinder_gl.h"
+
+#include <QImage>
+
+#include <libcamera/formats.h>
+
+static const QList<libcamera::PixelFormat> supportedFormats{
+ libcamera::formats::NV12,
+ libcamera::formats::NV21,
+ libcamera::formats::NV16,
+ libcamera::formats::NV61,
+ libcamera::formats::NV24,
+ libcamera::formats::NV42,
+ libcamera::formats::YUV420,
+ libcamera::formats::YVU420
+};
+
+ViewFinderGL::ViewFinderGL(QWidget *parent)
+ : QOpenGLWidget(parent), buffer_(nullptr), yuvData_(nullptr),
+ fragmentShader_(nullptr), vertexShader_(nullptr),
+ vertexBuffer_(QOpenGLBuffer::VertexBuffer),
+ textureU_(QOpenGLTexture::Target2D),
+ textureV_(QOpenGLTexture::Target2D),
+ textureY_(QOpenGLTexture::Target2D)
+{
+}
+
+ViewFinderGL::~ViewFinderGL()
+{
+ removeShader();
+
+ if (vertexBuffer_.isCreated())
+ vertexBuffer_.destroy();
+}
+
+const QList<libcamera::PixelFormat> &ViewFinderGL::nativeFormats() const
+{
+ return supportedFormats;
+}
+
+int ViewFinderGL::setFormat(const libcamera::PixelFormat &format,
+ const QSize &size)
+{
+ int ret = 0;
+
+ /* If the fragment is ceeated remove it and create a new one */
+ if (fragmentShader_) {
+ if (shaderProgram_.isLinked()) {
+ shaderProgram_.release();
+ shaderProgram_.removeShader(fragmentShader_);
+ delete fragmentShader_;
+ }
+ }
+
+ if (selectFormat(format)) {
+ format_ = format;
+ size_ = size;
+ } else {
+ ret = -1;
+ }
+ updateGeometry();
+ return ret;
+}
+
+void ViewFinderGL::stop()
+{
+ if (buffer_) {
+ renderComplete(buffer_);
+ buffer_ = nullptr;
+ }
+}
+
+QImage ViewFinderGL::getCurrentImage()
+{
+ QMutexLocker locker(&mutex_);
+
+ return grabFramebuffer();
+}
+
+void ViewFinderGL::render(libcamera::FrameBuffer *buffer, MappedBuffer *map)
+{
+ if (buffer->planes().size() != 1) {
+ qWarning() << "Multi-planar buffers are not supported";
+ return;
+ }
+
+ if (buffer_)
+ renderComplete(buffer_);
+
+ yuvData_ = static_cast<unsigned char *>(map->memory);
+ update();
+ buffer_ = buffer;
+}
+
+bool ViewFinderGL::selectFormat(const libcamera::PixelFormat &format)
+{
+ bool ret = true;
+ switch (format) {
+ case libcamera::formats::NV12:
+ horzSubSample_ = 2;
+ vertSubSample_ = 2;
+ vertexShaderSrc_ = ":NV_vertex_shader.glsl";
+ fragmentShaderSrc_ = ":NV_2_planes_UV_f.glsl";
+ break;
+ case libcamera::formats::NV21:
+ horzSubSample_ = 2;
+ vertSubSample_ = 2;
+ vertexShaderSrc_ = ":NV_vertex_shader.glsl";
+ fragmentShaderSrc_ = ":NV_2_planes_VU_f.glsl";
+ break;
+ case libcamera::formats::NV16:
+ horzSubSample_ = 2;
+ vertSubSample_ = 1;
+ vertexShaderSrc_ = ":NV_vertex_shader.glsl";
+ fragmentShaderSrc_ = ":NV_2_planes_UV_f.glsl";
+ break;
+ case libcamera::formats::NV61:
+ horzSubSample_ = 2;
+ vertSubSample_ = 1;
+ vertexShaderSrc_ = ":NV_vertex_shader.glsl";
+ fragmentShaderSrc_ = ":NV_2_planes_VU_f.glsl";
+ break;
+ case libcamera::formats::NV24:
+ horzSubSample_ = 1;
+ vertSubSample_ = 1;
+ vertexShaderSrc_ = ":NV_vertex_shader.glsl";
+ fragmentShaderSrc_ = ":NV_2_planes_UV_f.glsl";
+ break;
+ case libcamera::formats::NV42:
+ horzSubSample_ = 1;
+ vertSubSample_ = 1;
+ vertexShaderSrc_ = ":NV_vertex_shader.glsl";
+ fragmentShaderSrc_ = ":NV_2_planes_VU_f.glsl";
+ break;
+ case libcamera::formats::YUV420:
+ horzSubSample_ = 2;
+ vertSubSample_ = 2;
+ vertexShaderSrc_ = ":NV_vertex_shader.glsl";
+ fragmentShaderSrc_ = ":NV_3_planes_f.glsl";
+ break;
+ case libcamera::formats::YVU420:
+ horzSubSample_ = 2;
+ vertSubSample_ = 2;
+ vertexShaderSrc_ = ":NV_vertex_shader.glsl";
+ fragmentShaderSrc_ = ":NV_3_planes_f.glsl";
+ break;
+ default:
+ ret = false;
+ qWarning() << "[ViewFinderGL]:"
+ << "format not supported.";
+ break;
+ };
+
+ return ret;
+}
+
+bool ViewFinderGL::createVertexShader()
+{
+ /* Create Vertex Shader */
+ vertexShader_ = new QOpenGLShader(QOpenGLShader::Vertex, this);
+
+ /* Compile the vertex shader */
+ if (!vertexShader_->compileSourceFile(vertexShaderSrc_)) {
+ qWarning() << "[ViewFinderGL]:" << vertexShader_->log();
+ return false;
+ }
+
+ shaderProgram_.addShader(vertexShader_);
+ return true;
+}
+
+bool ViewFinderGL::createFragmentShader()
+{
+ int attributeVertex;
+ int attributeTexture;
+
+ /* Create Fragment Shader */
+ fragmentShader_ = new QOpenGLShader(QOpenGLShader::Fragment, this);
+
+ /* Compile the fragment shader */
+ if (!fragmentShader_->compileSourceFile(fragmentShaderSrc_)) {
+ qWarning() << "[ViewFinderGL]:" << fragmentShader_->log();
+ return false;
+ }
+
+ shaderProgram_.addShader(fragmentShader_);
+
+ /* Link shader pipeline */
+ if (!shaderProgram_.link()) {
+ qWarning() << "[ViewFinderGL]:" << shaderProgram_.log();
+ close();
+ }
+
+ /* Bind shader pipeline for use */
+ if (!shaderProgram_.bind()) {
+ qWarning() << "[ViewFinderGL]:" << shaderProgram_.log();
+ close();
+ }
+
+ attributeVertex = shaderProgram_.attributeLocation("vertexIn");
+ attributeTexture = shaderProgram_.attributeLocation("textureIn");
+
+ shaderProgram_.enableAttributeArray(attributeVertex);
+ shaderProgram_.setAttributeBuffer(attributeVertex,
+ GL_FLOAT,
+ 0,
+ 2,
+ 2 * sizeof(GLfloat));
+
+ shaderProgram_.enableAttributeArray(attributeTexture);
+ shaderProgram_.setAttributeBuffer(attributeTexture,
+ GL_FLOAT,
+ 8 * sizeof(GLfloat),
+ 2,
+ 2 * sizeof(GLfloat));
+
+ textureUniformY_ = shaderProgram_.uniformLocation("tex_y");
+ textureUniformU_ = shaderProgram_.uniformLocation("tex_u");
+ textureUniformV_ = shaderProgram_.uniformLocation("tex_v");
+
+ if (!textureY_.isCreated())
+ textureY_.create();
+
+ if (!textureU_.isCreated())
+ textureU_.create();
+
+ if (!textureV_.isCreated())
+ textureV_.create();
+
+ id_y_ = textureY_.textureId();
+ id_u_ = textureU_.textureId();
+ id_v_ = textureV_.textureId();
+ return true;
+}
+
+void ViewFinderGL::configureTexture(unsigned int id)
+{
+ glBindTexture(GL_TEXTURE_2D, id);
+ glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
+ glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
+ glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
+ glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
+}
+
+void ViewFinderGL::removeShader()
+{
+ if (shaderProgram_.isLinked()) {
+ shaderProgram_.release();
+ shaderProgram_.removeAllShaders();
+ }
+
+ if (fragmentShader_)
+ delete fragmentShader_;
+
+ if (vertexShader_)
+ delete vertexShader_;
+}
+
+void ViewFinderGL::initializeGL()
+{
+ initializeOpenGLFunctions();
+ glEnable(GL_TEXTURE_2D);
+ glDisable(GL_DEPTH_TEST);
+
+ static const GLfloat coordinates[2][4][2]{
+ {
+ /* Vertex coordinates */
+ { -1.0f, -1.0f },
+ { -1.0f, +1.0f },
+ { +1.0f, +1.0f },
+ { +1.0f, -1.0f },
+ },
+ {
+ /* Texture coordinates */
+ { 1.0f, 0.0f },
+ { 1.0f, 1.0f },
+ { 0.0f, 1.0f },
+ { 0.0f, 0.0f },
+ },
+ };
+
+ vertexBuffer_.create();
+ vertexBuffer_.bind();
+ vertexBuffer_.allocate(coordinates, sizeof(coordinates));
+
+ /* Create Vertex Shader */
+ if (!createVertexShader())
+ qWarning() << "[ViewFinderGL]: create vertex shader failed.";
+
+ glClearColor(1.0f, 1.0f, 1.0f, 0.0f);
+}
+
+void ViewFinderGL::doRender()
+{
+ switch (format_) {
+ case libcamera::formats::NV12:
+ case libcamera::formats::NV21:
+ case libcamera::formats::NV16:
+ case libcamera::formats::NV61:
+ case libcamera::formats::NV24:
+ case libcamera::formats::NV42:
+ /* activate texture Y */
+ glActiveTexture(GL_TEXTURE0);
+ configureTexture(id_y_);
+ glTexImage2D(GL_TEXTURE_2D,
+ 0,
+ GL_RED,
+ size_.width(),
+ size_.height(),
+ 0,
+ GL_RED,
+ GL_UNSIGNED_BYTE,
+ yuvData_);
+ shaderProgram_.setUniformValue(textureUniformY_, 0);
+
+ /* activate texture UV/VU */
+ glActiveTexture(GL_TEXTURE1);
+ configureTexture(id_u_);
+ glTexImage2D(GL_TEXTURE_2D,
+ 0,
+ GL_RG,
+ size_.width() / horzSubSample_,
+ size_.height() / vertSubSample_,
+ 0,
+ GL_RG,
+ GL_UNSIGNED_BYTE,
+ (char *)yuvData_ + size_.width() * size_.height());
+ shaderProgram_.setUniformValue(textureUniformU_, 1);
+ break;
+
+ case libcamera::formats::YUV420:
+ /* activate texture Y */
+ glActiveTexture(GL_TEXTURE0);
+ configureTexture(id_y_);
+ glTexImage2D(GL_TEXTURE_2D,
+ 0,
+ GL_RED,
+ size_.width(),
+ size_.height(),
+ 0,
+ GL_RED,
+ GL_UNSIGNED_BYTE,
+ yuvData_);
+ shaderProgram_.setUniformValue(textureUniformY_, 0);
+
+ /* activate texture U */
+ glActiveTexture(GL_TEXTURE1);
+ configureTexture(id_u_);
+ glTexImage2D(GL_TEXTURE_2D,
+ 0,
+ GL_RED,
+ size_.width() / horzSubSample_,
+ size_.height() / vertSubSample_,
+ 0,
+ GL_RED,
+ GL_UNSIGNED_BYTE,
+ (char *)yuvData_ + size_.width() * size_.height());
+ shaderProgram_.setUniformValue(textureUniformU_, 1);
+
+ /* activate texture V */
+ glActiveTexture(GL_TEXTURE2);
+ configureTexture(id_v_);
+ glTexImage2D(GL_TEXTURE_2D,
+ 0,
+ GL_RED,
+ size_.width() / horzSubSample_,
+ size_.height() / vertSubSample_,
+ 0,
+ GL_RED,
+ GL_UNSIGNED_BYTE,
+ (char *)yuvData_ + size_.width() * size_.height() * 5 / 4);
+ shaderProgram_.setUniformValue(textureUniformV_, 2);
+ break;
+
+ case libcamera::formats::YVU420:
+ /* activate texture Y */
+ glActiveTexture(GL_TEXTURE0);
+ configureTexture(id_y_);
+ glTexImage2D(GL_TEXTURE_2D,
+ 0,
+ GL_RED,
+ size_.width(),
+ size_.height(),
+ 0,
+ GL_RED,
+ GL_UNSIGNED_BYTE,
+ yuvData_);
+ shaderProgram_.setUniformValue(textureUniformY_, 0);
+
+ /* activate texture V */
+ glActiveTexture(GL_TEXTURE2);
+ configureTexture(id_v_);
+ glTexImage2D(GL_TEXTURE_2D,
+ 0,
+ GL_RED,
+ size_.width() / horzSubSample_,
+ size_.height() / vertSubSample_,
+ 0,
+ GL_RED,
+ GL_UNSIGNED_BYTE,
+ (char *)yuvData_ + size_.width() * size_.height());
+ shaderProgram_.setUniformValue(textureUniformV_, 1);
+
+ /* activate texture U */
+ glActiveTexture(GL_TEXTURE1);
+ configureTexture(id_u_);
+ glTexImage2D(GL_TEXTURE_2D,
+ 0,
+ GL_RED,
+ size_.width() / horzSubSample_,
+ size_.height() / vertSubSample_,
+ 0,
+ GL_RED,
+ GL_UNSIGNED_BYTE,
+ (char *)yuvData_ + size_.width() * size_.height() * 5 / 4);
+ shaderProgram_.setUniformValue(textureUniformU_, 2);
+ break;
+
+ default:
+ break;
+ };
+}
+
+void ViewFinderGL::paintGL()
+{
+ if (!fragmentShader_)
+ if (!createFragmentShader()) {
+ qWarning() << "[ViewFinderGL]:"
+ << "create fragment shader failed.";
+ }
+
+ if (yuvData_) {
+ glClearColor(0.0, 0.0, 0.0, 1.0);
+ glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
+
+ doRender();
+ glDrawArrays(GL_TRIANGLE_FAN, 0, 4);
+ }
+}
+
+void ViewFinderGL::resizeGL(int w, int h)
+{
+ glViewport(0, 0, w, h);
+}
+
+QSize ViewFinderGL::sizeHint() const
+{
+ return size_.isValid() ? size_ : QSize(640, 480);
+}
diff --git a/src/qcam/viewfinder_gl.h b/src/qcam/viewfinder_gl.h
new file mode 100644
index 0000000..69502b7
--- /dev/null
+++ b/src/qcam/viewfinder_gl.h
@@ -0,0 +1,96 @@
+/* SPDX-License-Identifier: LGPL-2.1-or-later */
+/*
+ * Copyright (C) 2020, Linaro
+ *
+ * viewfinder_GL.h - OpenGL Viewfinder for rendering by OpenGL shader
+ *
+ */
+#ifndef __VIEWFINDER_GL_H__
+#define __VIEWFINDER_GL_H__
+
+#include <QImage>
+#include <QMutex>
+#include <QOpenGLBuffer>
+#include <QOpenGLFunctions>
+#include <QOpenGLShader>
+#include <QOpenGLShaderProgram>
+#include <QOpenGLTexture>
+#include <QOpenGLWidget>
+#include <QSize>
+
+#include <libcamera/buffer.h>
+#include <libcamera/formats.h>
+
+#include "viewfinder.h"
+
+class ViewFinderGL : public QOpenGLWidget,
+ public ViewFinder,
+ protected QOpenGLFunctions
+{
+ Q_OBJECT
+
+public:
+ ViewFinderGL(QWidget *parent = nullptr);
+ ~ViewFinderGL();
+
+ const QList<libcamera::PixelFormat> &nativeFormats() const override;
+
+ int setFormat(const libcamera::PixelFormat &format, const QSize &size) override;
+ void render(libcamera::FrameBuffer *buffer, MappedBuffer *map) override;
+ void stop() override;
+
+ QImage getCurrentImage() override;
+
+Q_SIGNALS:
+ void renderComplete(libcamera::FrameBuffer *buffer);
+
+protected:
+ void initializeGL() override;
+ void paintGL() override;
+ void resizeGL(int w, int h) override;
+ QSize sizeHint() const override;
+
+private:
+ bool selectFormat(const libcamera::PixelFormat &format);
+
+ void configureTexture(unsigned int id);
+ bool createFragmentShader();
+ bool createVertexShader();
+ void removeShader();
+ void doRender();
+
+ /* Captured image size, format and buffer */
+ libcamera::FrameBuffer *buffer_;
+ libcamera::PixelFormat format_;
+ QSize size_;
+ unsigned char *yuvData_;
+
+ /* OpenGL components for rendering */
+ QOpenGLShader *fragmentShader_;
+ QOpenGLShader *vertexShader_;
+ QOpenGLShaderProgram shaderProgram_;
+
+ /* Vertex buffer */
+ QOpenGLBuffer vertexBuffer_;
+
+ /* Fragment and Vertex shader file name */
+ QString fragmentShaderSrc_;
+ QString vertexShaderSrc_;
+
+ /* YUV texture planars and parameters */
+ GLuint id_u_;
+ GLuint id_v_;
+ GLuint id_y_;
+ GLuint textureUniformU_;
+ GLuint textureUniformV_;
+ GLuint textureUniformY_;
+ QOpenGLTexture textureU_;
+ QOpenGLTexture textureV_;
+ QOpenGLTexture textureY_;
+ unsigned int horzSubSample_;
+ unsigned int vertSubSample_;
+
+ QMutex mutex_; /* Prevent concurrent access to image_ */
+};
+
+#endif /* __VIEWFINDER_GL_H__ */
--
2.20.1
More information about the libcamera-devel
mailing list