[PATCH v2] gstreamer: Add GstVideoMeta support

Kieran Bingham kieran.bingham at ideasonboard.com
Mon Nov 11 10:30:13 CET 2024


Hi Qi,

Quoting Hou Qi (2024-11-08 09:21:13)
> GStreamer video-info calculated stride and offset may differ from
> those used by the camera.
> 
> This patch enhances downstream plugin's support for videometa by
> adding videometa support for libcamerasrc. It ensures that when
> downstream plugin supports videometa by allocation query, libcamerasrc
> also attaches videometa to buffer, preventing discrepancies
> between the stride and offset calculated by video-info and camera.

Thanks for the v2, this looks like interesting work.

Unfortunately the CI isn't clear on this one yet:

https://gitlab.freedesktop.org/camera/libcamera/-/jobs/66327653

Any idea what's happening here? It looks like this only failed on one
compiler:

[443/641] Generating src/py/libcamera/py_gen_formats with a custom command
[444/641] Compiling C++ object src/gstreamer/libgstlibcamera.so.p/gstlibcamerapool.cpp.o
FAILED: src/gstreamer/libgstlibcamera.so.p/gstlibcamerapool.cpp.o
g++-9 -Isrc/gstreamer/libgstlibcamera.so.p -Isrc/gstreamer -I../src/gstreamer -Iinclude -I../include -Iinclude/libcamera -I/usr/include/gstreamer-1.0 -I/usr/include/orc-0.4 -I/usr/include/x86_64-linux-gnu -I/usr/include/glib-2.0 -I/usr/lib/x86_64-linux-gnu/glib-2.0/include -fdiagnostics-color=always -D_FILE_OFFSET_BITS=64 -Wall -Winvalid-pch -Wextra -Werror -std=c++17 -O0 -g -Wmissing-declarations -Wshadow -include /builds/camera/libcamera/build/config.h -fPIC -pthread '-DVERSION="0.3.2+1-edf89091-nvm"' '-DPACKAGE="libcamera"' -DGLIB_VERSION_MIN_REQUIRED=GLIB_VERSION_2_40 -MD -MQ src/gstreamer/libgstlibcamera.so.p/gstlibcamerapool.cpp.o -MF src/gstreamer/libgstlibcamera.so.p/gstlibcamerapool.cpp.o.d -o src/gstreamer/libgstlibcamera.so.p/gstlibcamerapool.cpp.o -c ../src/gstreamer/gstlibcamerapool.cpp
../src/gstreamer/gstlibcamerapool.cpp: In function ‘GstLibcameraPool* gst_libcamera_pool_new(GstLibcameraAllocator*, const libcamera::StreamConfiguration&, GstCaps*, gboolean)’:
../src/gstreamer/gstlibcamerapool.cpp:151:13: error: ‘gst_video_format_info_extrapolate_stride’ was not declared in this scope; did you mean ‘gst_video_format_info_component’?
  151 |    stride = gst_video_format_info_extrapolate_stride(info.finfo, k, stream_cfg.stride);
      |             ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
      |             gst_video_format_info_component
[445/641] Compiling C++ object src/apps/qcam/qcam.p/viewfinder_qt.cpp.o

So perhaps we're hitting a requirement to have a specific gstreamer
version that isn't being catered for ?

--
Kieran


> 
> Signed-off-by: Hou Qi <qi.hou at nxp.com>
> ---
>  src/gstreamer/gstlibcamerapool.cpp | 28 ++++++++++++++++++++++++----
>  src/gstreamer/gstlibcamerapool.h   |  2 +-
>  src/gstreamer/gstlibcamerasrc.cpp  | 14 +++++++++++++-
>  3 files changed, 38 insertions(+), 6 deletions(-)
> 
> diff --git a/src/gstreamer/gstlibcamerapool.cpp b/src/gstreamer/gstlibcamerapool.cpp
> index 9cd7eccb..6593b3ca 100644
> --- a/src/gstreamer/gstlibcamerapool.cpp
> +++ b/src/gstreamer/gstlibcamerapool.cpp
> @@ -135,16 +135,36 @@ gst_libcamera_pool_class_init(GstLibcameraPoolClass *klass)
>  }
>  
>  GstLibcameraPool *
> -gst_libcamera_pool_new(GstLibcameraAllocator *allocator, Stream *stream)
> +gst_libcamera_pool_new(GstLibcameraAllocator *allocator, const StreamConfiguration &stream_cfg,
> +                      GstCaps *caps, gboolean add_video_meta)
>  {
>         auto *pool = GST_LIBCAMERA_POOL(g_object_new(GST_TYPE_LIBCAMERA_POOL, nullptr));
> +       GstVideoInfo info;
>  
>         pool->allocator = GST_LIBCAMERA_ALLOCATOR(g_object_ref(allocator));
> -       pool->stream = stream;
> -
> -       gsize pool_size = gst_libcamera_allocator_get_pool_size(allocator, stream);
> +       pool->stream = stream_cfg.stream();
> +
> +       if (caps && gst_video_info_from_caps(&info, caps)) {
> +               guint k, stride;
> +               gsize offset = 0;
> +               for (k = 0; k < GST_VIDEO_INFO_N_PLANES(&info); k++) {
> +                       stride = gst_video_format_info_extrapolate_stride(info.finfo, k, stream_cfg.stride);
> +                       info.stride[k] = stride;
> +                       info.offset[k] = offset;
> +                       offset += stride * GST_VIDEO_FORMAT_INFO_SCALE_HEIGHT(info.finfo, k, GST_VIDEO_INFO_HEIGHT(&info));
> +               }
> +       } else
> +               add_video_meta = false;
> +
> +       gsize pool_size = gst_libcamera_allocator_get_pool_size(allocator, stream_cfg.stream());
>         for (gsize i = 0; i < pool_size; i++) {
>                 GstBuffer *buffer = gst_buffer_new();
> +               if (add_video_meta) {
> +                       gst_buffer_add_video_meta_full(buffer, GST_VIDEO_FRAME_FLAG_NONE,
> +                                                      GST_VIDEO_INFO_FORMAT(&info), GST_VIDEO_INFO_WIDTH(&info),
> +                                                      GST_VIDEO_INFO_HEIGHT(&info), GST_VIDEO_INFO_N_PLANES(&info),
> +                                                      info.offset, info.stride);
> +               }
>                 pool->queue->push_back(buffer);
>         }
>  
> diff --git a/src/gstreamer/gstlibcamerapool.h b/src/gstreamer/gstlibcamerapool.h
> index 2a7a9c77..8ad87cab 100644
> --- a/src/gstreamer/gstlibcamerapool.h
> +++ b/src/gstreamer/gstlibcamerapool.h
> @@ -21,7 +21,7 @@
>  G_DECLARE_FINAL_TYPE(GstLibcameraPool, gst_libcamera_pool, GST_LIBCAMERA, POOL, GstBufferPool)
>  
>  GstLibcameraPool *gst_libcamera_pool_new(GstLibcameraAllocator *allocator,
> -                                        libcamera::Stream *stream);
> +                                        const libcamera::StreamConfiguration &stream_cfg, GstCaps *caps, gboolean add_video_meta);
>  
>  libcamera::Stream *gst_libcamera_pool_get_stream(GstLibcameraPool *self);
>  
> diff --git a/src/gstreamer/gstlibcamerasrc.cpp b/src/gstreamer/gstlibcamerasrc.cpp
> index 8efa25f4..c05a31e7 100644
> --- a/src/gstreamer/gstlibcamerasrc.cpp
> +++ b/src/gstreamer/gstlibcamerasrc.cpp
> @@ -497,9 +497,21 @@ gst_libcamera_src_negotiate(GstLibcameraSrc *self)
>         for (gsize i = 0; i < state->srcpads_.size(); i++) {
>                 GstPad *srcpad = state->srcpads_[i];
>                 const StreamConfiguration &stream_cfg = state->config_->at(i);
> +               GstQuery *query = NULL;
> +               gboolean add_video_meta = false;
> +
> +               g_autoptr(GstCaps) caps = gst_libcamera_stream_configuration_to_caps(stream_cfg);
> +               gst_libcamera_framerate_to_caps(caps, element_caps);
> +
> +               query = gst_query_new_allocation(caps, false);
> +               if (!gst_pad_peer_query(srcpad, query))
> +                       GST_DEBUG_OBJECT(self, "didn't get downstream ALLOCATION hints");
> +               else
> +                       add_video_meta = gst_query_find_allocation_meta(query, GST_VIDEO_META_API_TYPE, NULL);
> +               gst_query_unref(query);
>  
>                 GstLibcameraPool *pool = gst_libcamera_pool_new(self->allocator,
> -                                                               stream_cfg.stream());
> +                                                               stream_cfg, caps, add_video_meta);
>                 g_signal_connect_swapped(pool, "buffer-notify",
>                                          G_CALLBACK(gst_task_resume), self->task);
>  
> -- 
> 2.34.1
>


More information about the libcamera-devel mailing list