[PATCH v4] gstreamer: Add GstVideoMeta support
Hou Qi
qi.hou at nxp.com
Wed Nov 20 11:04:14 CET 2024
GStreamer video-info calculated stride and offset may differ from
those used by the camera.
This patch enhances downstream plugin's support for videometa by
adding videometa support for libcamerasrc. It ensures that when
downstream plugin supports videometa by allocation query, libcamerasrc
also attaches videometa to buffer, preventing discrepancies
between the stride and offset calculated by video-info and camera.
Signed-off-by: Hou Qi <qi.hou at nxp.com>
---
src/gstreamer/gstlibcamera-utils.cpp | 33 ++++++++++++++++
src/gstreamer/gstlibcamera-utils.h | 4 ++
src/gstreamer/gstlibcamerapool.cpp | 57 +++++++++++++++++++++++++++-
src/gstreamer/gstlibcamerapool.h | 5 ++-
src/gstreamer/gstlibcamerasrc.cpp | 47 +++++++++++++++++++++--
5 files changed, 141 insertions(+), 5 deletions(-)
diff --git a/src/gstreamer/gstlibcamera-utils.cpp b/src/gstreamer/gstlibcamera-utils.cpp
index 732987ef..09af9204 100644
--- a/src/gstreamer/gstlibcamera-utils.cpp
+++ b/src/gstreamer/gstlibcamera-utils.cpp
@@ -591,6 +591,39 @@ gst_task_resume(GstTask *task)
}
#endif
+#if !GST_CHECK_VERSION(1, 22, 0)
+/*
+ * Copyright (C) <1999> Erik Walthinsen <omega at cse.ogi.edu>
+ * Library <2002> Ronald Bultje <rbultje at ronald.bitfreak.net>
+ * Copyright (C) <2007> David A. Schleef <ds at schleef.org>
+ */
+/* This function has been imported directly from the gstreamer project to
+ * support backwards compatibility and should be removed when the older
+ * version is no longer supported. */
+gint gst_video_format_info_extrapolate_stride(const GstVideoFormatInfo *finfo, gint plane, gint stride)
+{
+ gint estride;
+ gint comp[GST_VIDEO_MAX_COMPONENTS];
+ gint i;
+
+ /* there is nothing to extrapolate on first plane */
+ if (plane == 0)
+ return stride;
+
+ gst_video_format_info_component(finfo, plane, comp);
+
+ /* For now, all planar formats have a single component on first plane, but
+ * if there was a planar format with more, we'd have to make a ratio of the
+ * number of component on the first plane against the number of component on
+ * the current plane. */
+ estride = 0;
+ for (i = 0; i < GST_VIDEO_MAX_COMPONENTS && comp[i] >= 0; i++)
+ estride += GST_VIDEO_FORMAT_INFO_SCALE_WIDTH(finfo, comp[i], stride);
+
+ return estride;
+}
+#endif
+
G_LOCK_DEFINE_STATIC(cm_singleton_lock);
static std::weak_ptr<CameraManager> cm_singleton_ptr;
diff --git a/src/gstreamer/gstlibcamera-utils.h b/src/gstreamer/gstlibcamera-utils.h
index cab1c814..81149280 100644
--- a/src/gstreamer/gstlibcamera-utils.h
+++ b/src/gstreamer/gstlibcamera-utils.h
@@ -35,6 +35,10 @@ static inline void gst_clear_event(GstEvent **event_ptr)
#if !GST_CHECK_VERSION(1, 17, 1)
gboolean gst_task_resume(GstTask *task);
#endif
+
+#if !GST_CHECK_VERSION(1, 22, 0)
+gint gst_video_format_info_extrapolate_stride(const GstVideoFormatInfo *finfo, gint plane, gint stride);
+#endif
std::shared_ptr<libcamera::CameraManager> gst_libcamera_get_camera_manager(int &ret);
/**
diff --git a/src/gstreamer/gstlibcamerapool.cpp b/src/gstreamer/gstlibcamerapool.cpp
index 9cd7eccb..bd1cc473 100644
--- a/src/gstreamer/gstlibcamerapool.cpp
+++ b/src/gstreamer/gstlibcamerapool.cpp
@@ -29,6 +29,8 @@ struct _GstLibcameraPool {
std::deque<GstBuffer *> *queue;
GstLibcameraAllocator *allocator;
Stream *stream;
+ gboolean frame_copy;
+ GstVideoInfo info;
};
G_DEFINE_TYPE(GstLibcameraPool, gst_libcamera_pool, GST_TYPE_BUFFER_POOL)
@@ -135,16 +137,29 @@ gst_libcamera_pool_class_init(GstLibcameraPoolClass *klass)
}
GstLibcameraPool *
-gst_libcamera_pool_new(GstLibcameraAllocator *allocator, Stream *stream)
+gst_libcamera_pool_new(GstLibcameraAllocator *allocator, Stream *stream,
+ GstVideoInfo *info, gboolean stride_mismatch, gboolean has_video_meta)
{
auto *pool = GST_LIBCAMERA_POOL(g_object_new(GST_TYPE_LIBCAMERA_POOL, nullptr));
pool->allocator = GST_LIBCAMERA_ALLOCATOR(g_object_ref(allocator));
pool->stream = stream;
+ pool->info = *info;
+
+ if (stride_mismatch && !has_video_meta)
+ pool->frame_copy = true;
+ else
+ pool->frame_copy = false;
gsize pool_size = gst_libcamera_allocator_get_pool_size(allocator, stream);
for (gsize i = 0; i < pool_size; i++) {
GstBuffer *buffer = gst_buffer_new();
+ if (stride_mismatch && has_video_meta) {
+ gst_buffer_add_video_meta_full(buffer, GST_VIDEO_FRAME_FLAG_NONE,
+ GST_VIDEO_INFO_FORMAT(info), GST_VIDEO_INFO_WIDTH(info),
+ GST_VIDEO_INFO_HEIGHT(info), GST_VIDEO_INFO_N_PLANES(info),
+ info->offset, info->stride);
+ }
pool->queue->push_back(buffer);
}
@@ -163,3 +178,43 @@ gst_libcamera_buffer_get_frame_buffer(GstBuffer *buffer)
GstMemory *mem = gst_buffer_peek_memory(buffer, 0);
return gst_libcamera_memory_get_frame_buffer(mem);
}
+
+GstBuffer *
+gst_libcamera_copy_buffer(GstLibcameraPool *self, GstBuffer *src, FrameBuffer *fb, guint32 stride)
+{
+ if (self->frame_copy) {
+ GstVideoInfo src_info = self->info;
+ gsize size = GST_VIDEO_INFO_SIZE(&self->info);
+ GstBuffer *dest = gst_buffer_new_allocate(NULL, size, NULL);
+ GstVideoFrame src_frame, dest_frame;
+ int i = 0;
+
+ for (const FrameBuffer::Plane &plane : fb->planes()) {
+ src_info.stride[i] = stride;
+ src_info.offset[i] = plane.offset;
+ i++;
+ }
+ src_info.size = gst_buffer_get_size(src);
+
+ if (!gst_video_frame_map(&src_frame, &src_info, src, GST_MAP_READ)) {
+ GST_WARNING("fail to map src_frame");
+ return src;
+ }
+
+ if (!gst_video_frame_map(&dest_frame, &self->info, dest, GST_MAP_WRITE)) {
+ gst_video_frame_unmap(&src_frame);
+ GST_WARNING("fail to map dest_frame");
+ return src;
+ }
+
+ gst_video_frame_copy(&dest_frame, &src_frame);
+
+ gst_video_frame_unmap(&src_frame);
+ gst_video_frame_unmap(&dest_frame);
+
+ gst_buffer_unref(src);
+ return dest;
+ }
+
+ return src;
+}
diff --git a/src/gstreamer/gstlibcamerapool.h b/src/gstreamer/gstlibcamerapool.h
index 2a7a9c77..ffbd3173 100644
--- a/src/gstreamer/gstlibcamerapool.h
+++ b/src/gstreamer/gstlibcamerapool.h
@@ -14,6 +14,7 @@
#include "gstlibcameraallocator.h"
#include <gst/gst.h>
+#include <gst/video/video.h>
#include <libcamera/stream.h>
@@ -21,8 +22,10 @@
G_DECLARE_FINAL_TYPE(GstLibcameraPool, gst_libcamera_pool, GST_LIBCAMERA, POOL, GstBufferPool)
GstLibcameraPool *gst_libcamera_pool_new(GstLibcameraAllocator *allocator,
- libcamera::Stream *stream);
+ libcamera::Stream *stream, GstVideoInfo *info, gboolean stride_mismatch, gboolean has_video_meta);
libcamera::Stream *gst_libcamera_pool_get_stream(GstLibcameraPool *self);
libcamera::FrameBuffer *gst_libcamera_buffer_get_frame_buffer(GstBuffer *buffer);
+
+GstBuffer *gst_libcamera_copy_buffer(GstLibcameraPool *self, GstBuffer *buffer, libcamera::FrameBuffer *fb, guint32 stride);
diff --git a/src/gstreamer/gstlibcamerasrc.cpp b/src/gstreamer/gstlibcamerasrc.cpp
index 8efa25f4..1d46536a 100644
--- a/src/gstreamer/gstlibcamerasrc.cpp
+++ b/src/gstreamer/gstlibcamerasrc.cpp
@@ -292,12 +292,17 @@ int GstLibcameraSrcState::processRequest()
GstFlowReturn ret = GST_FLOW_OK;
gst_flow_combiner_reset(src_->flow_combiner);
- for (GstPad *srcpad : srcpads_) {
+ for (gsize i = 0; i < src_->state->srcpads_.size(); i++) {
+ GstPad *srcpad = src_->state->srcpads_[i];
Stream *stream = gst_libcamera_pad_get_stream(srcpad);
GstBuffer *buffer = wrap->detachBuffer(stream);
+ const StreamConfiguration &stream_cfg = src_->state->config_->at(i);
+ GstLibcameraPool *pool = gst_libcamera_pad_get_pool(srcpad);
FrameBuffer *fb = gst_libcamera_buffer_get_frame_buffer(buffer);
+ buffer = gst_libcamera_copy_buffer(pool, buffer, fb, stream_cfg.stride);
+
if (GST_CLOCK_TIME_IS_VALID(wrap->pts_)) {
GST_BUFFER_PTS(buffer) = wrap->pts_;
gst_libcamera_pad_set_latency(srcpad, wrap->latency_);
@@ -497,9 +502,45 @@ gst_libcamera_src_negotiate(GstLibcameraSrc *self)
for (gsize i = 0; i < state->srcpads_.size(); i++) {
GstPad *srcpad = state->srcpads_[i];
const StreamConfiguration &stream_cfg = state->config_->at(i);
+ gboolean stride_mismatch = false, has_video_meta = false;
+ GstVideoInfo info;
+
+ g_autoptr(GstCaps) caps = gst_libcamera_stream_configuration_to_caps(stream_cfg);
+ gst_libcamera_framerate_to_caps(caps, element_caps);
+
+ gst_video_info_init(&info);
+ gst_video_info_from_caps(&info, caps);
+
+ /* stride mismatch between camera stride and that calculated by video-info */
+ if (static_cast<unsigned int>(info.stride[0]) != stream_cfg.stride &&
+ GST_VIDEO_INFO_FORMAT(&info) != GST_VIDEO_FORMAT_ENCODED) {
+ GstQuery *query = NULL;
+ gboolean need_pool = false;
+ stride_mismatch = true;
+
+ query = gst_query_new_allocation(caps, need_pool);
+ if (!gst_pad_peer_query(srcpad, query))
+ GST_DEBUG_OBJECT(self, "didn't get downstream ALLOCATION hints");
+ else
+ has_video_meta = gst_query_find_allocation_meta(query, GST_VIDEO_META_API_TYPE, NULL);
+ gst_query_unref(query);
+
+ if (has_video_meta) {
+ guint k, stride;
+ gsize offset = 0;
+
+ /* this should be updated if tiled formats get added in the future. */
+ for (k = 0; k < GST_VIDEO_INFO_N_PLANES(&info); k++) {
+ stride = gst_video_format_info_extrapolate_stride(info.finfo, k, stream_cfg.stride);
+ info.stride[k] = stride;
+ info.offset[k] = offset;
+ offset += stride * GST_VIDEO_FORMAT_INFO_SCALE_HEIGHT(info.finfo, k, GST_VIDEO_INFO_HEIGHT(&info));
+ }
+ }
+ }
- GstLibcameraPool *pool = gst_libcamera_pool_new(self->allocator,
- stream_cfg.stream());
+ GstLibcameraPool *pool = gst_libcamera_pool_new(self->allocator, stream_cfg.stream(),
+ &info, stride_mismatch, has_video_meta);
g_signal_connect_swapped(pool, "buffer-notify",
G_CALLBACK(gst_task_resume), self->task);
--
2.34.1
More information about the libcamera-devel
mailing list