[libcamera-devel] [PATCH v1 1/3] gstreamer: Add sensor mode selection
Nicolas Dufresne
nicolas.dufresne at collabora.com
Fri Mar 24 19:03:35 CET 2023
Le vendredi 24 mars 2023 à 13:40 -0400, Nicolas Dufresne a écrit :
> From: Nicolas Dufresne <nicolas.dufresne at collabora.com>
>
> This add support for selecting the sensor mode. A new read-only
> property called sensor-modes is filled when the element reaches
> READY state. It contains the list of all available sensor modes,
> including the supported framerate range. This is exposed as GstCaps
> in the form of sensor/mode,width=X,height=Y,format=Y,framerate=[...].
> The format string matches the libcamera format string representation.
>
> The application can then select a mode using the read/write sensor-mode
> control. The selected mode is also a caps, it will be intersected with
> the supported mode and the "best" match will be picked. This allows
> application to use simple filter when they want to pick a mode for lets
> say a specific framerate (e.g. sensor/mode,framerate=60/1).
>
> Signed-off-by: Nicolas Dufresne <nicolas.dufresne at collabora.com>
> ---
> src/gstreamer/gstlibcamera-utils.cpp | 4 +
> src/gstreamer/gstlibcamerasrc.cpp | 123 ++++++++++++++++++++++++++-
> 2 files changed, 126 insertions(+), 1 deletion(-)
>
> diff --git a/src/gstreamer/gstlibcamera-utils.cpp b/src/gstreamer/gstlibcamera-utils.cpp
> index 750ec351..c8a8df09 100644
> --- a/src/gstreamer/gstlibcamera-utils.cpp
> +++ b/src/gstreamer/gstlibcamera-utils.cpp
> @@ -416,6 +416,10 @@ gst_libcamera_configure_stream_from_caps(StreamConfiguration &stream_cfg,
> stream_cfg.pixelFormat = gst_format_to_pixel_format(gst_format);
> } else if (gst_structure_has_name(s, "image/jpeg")) {
> stream_cfg.pixelFormat = formats::MJPEG;
> + } else if (gst_structure_has_name(s, "sensor/mode")) {
> + gst_structure_fixate_field(s, "format");
> + const gchar *format = gst_structure_get_string(s, "format");
> + stream_cfg.pixelFormat = PixelFormat::fromString(format);
> } else {
> g_critical("Unsupported media type: %s", gst_structure_get_name(s));
> }
> diff --git a/src/gstreamer/gstlibcamerasrc.cpp b/src/gstreamer/gstlibcamerasrc.cpp
> index a10cbd4f..c448a9fe 100644
> --- a/src/gstreamer/gstlibcamerasrc.cpp
> +++ b/src/gstreamer/gstlibcamerasrc.cpp
> @@ -147,6 +147,15 @@ struct _GstLibcameraSrc {
>
> gchar *camera_name;
>
> +<<<<<<< HEAD
> +=======
My apology, scratch this please.
> + gboolean hflip;
> + gboolean vflip;
> +
> + GstCaps *sensor_modes;
> + GstCaps *sensor_mode;
> +
> +>>>>>>> 210f0a67 (gstreamer: Add sensor mode selection)
> GstLibcameraSrcState *state;
> GstLibcameraAllocator *allocator;
> GstFlowCombiner *flow_combiner;
> @@ -154,7 +163,16 @@ struct _GstLibcameraSrc {
>
> enum {
> PROP_0,
> +<<<<<<< HEAD
> PROP_CAMERA_NAME
> +=======
> + PROP_CAMERA_NAME,
> + PROP_HFLIP,
> + PROP_VFLIP,
> + PROP_SENSOR_MODES,
> + PROP_SENSOR_MODE,
> + PROP_LAST
> +>>>>>>> 210f0a67 (gstreamer: Add sensor mode selection)
> };
>
> G_DEFINE_TYPE_WITH_CODE(GstLibcameraSrc, gst_libcamera_src, GST_TYPE_ELEMENT,
> @@ -318,6 +336,61 @@ int GstLibcameraSrcState::processRequest()
> return err;
> }
>
> +static GstCaps *
> +gst_libcamera_src_enumerate_sensor_modes(GstLibcameraSrc *self)
> +{
> + GstCaps *modes = gst_caps_new_empty();
> + GstLibcameraSrcState *state = self->state;
> + auto config = state->cam_->generateConfiguration({ libcamera::StreamRole::Raw,
> + libcamera::StreamRole::VideoRecording });
> + if (config == nullptr) {
> + GST_DEBUG_OBJECT(self, "Sensor mode selection is not supported, skipping enumeration.");
> + return modes;
> + }
> +
> + const libcamera::StreamFormats &formats = config->at(0).formats();
> +
> + for (const auto &pixfmt : formats.pixelformats()) {
> + for (const auto &size : formats.sizes(pixfmt)) {
> + config->at(0).size = size;
> + config->at(0).pixelFormat = pixfmt;
> +
> + if (config->validate() == CameraConfiguration::Invalid)
> + continue;
> +
> + if (state->cam_->configure(config.get()))
> + continue;
> +
> + auto fd_ctrl = state->cam_->controls().find(&controls::FrameDurationLimits);
> + if (fd_ctrl == state->cam_->controls().end())
> + continue;
> +
> + int minrate_num, minrate_denom;
> + int maxrate_num, maxrate_denom;
> + double min_framerate = gst_util_guint64_to_gdouble(1.0e6) /
> + gst_util_guint64_to_gdouble(fd_ctrl->second.max().get<int64_t>());
> + double max_framerate = gst_util_guint64_to_gdouble(1.0e6) /
> + gst_util_guint64_to_gdouble(fd_ctrl->second.min().get<int64_t>());
> + gst_util_double_to_fraction(min_framerate, &minrate_num, &minrate_denom);
> + gst_util_double_to_fraction(max_framerate, &maxrate_num, &maxrate_denom);
> +
> + GstStructure *s = gst_structure_new("sensor/mode",
> + "format", G_TYPE_STRING, pixfmt.toString().c_str(),
> + "width", G_TYPE_INT, size.width,
> + "height", G_TYPE_INT, size.height,
> + "framerate", GST_TYPE_FRACTION_RANGE,
> + minrate_num, minrate_denom,
> + maxrate_num, maxrate_denom,
> + nullptr);
> + gst_caps_append_structure(modes, s);
> + }
> + }
> +
> + GST_DEBUG_OBJECT(self, "Camera sensor modes: %" GST_PTR_FORMAT, modes);
> +
> + return modes;
> +}
> +
> static bool
> gst_libcamera_src_open(GstLibcameraSrc *self)
> {
> @@ -375,6 +448,7 @@ gst_libcamera_src_open(GstLibcameraSrc *self)
> /* No need to lock here, we didn't start our threads yet. */
> self->state->cm_ = cm;
> self->state->cam_ = cam;
> + self->sensor_modes = gst_libcamera_src_enumerate_sensor_modes(self);
>
> return true;
> }
> @@ -462,6 +536,7 @@ gst_libcamera_src_task_enter(GstTask *task, [[maybe_unused]] GThread *thread,
> GstLibcameraSrcState *state = self->state;
> GstFlowReturn flow_ret = GST_FLOW_OK;
> gint ret;
> + g_autoptr(GstCaps) sensor_mode = nullptr;
>
> g_autoptr(GstStructure) element_caps = gst_structure_new_empty("caps");
>
> @@ -481,6 +556,16 @@ gst_libcamera_src_task_enter(GstTask *task, [[maybe_unused]] GThread *thread,
> roles.push_back(gst_libcamera_pad_get_role(srcpad));
> }
>
> + if (!gst_caps_is_any(self->sensor_mode)) {
> + sensor_mode = gst_caps_intersect(self->sensor_mode, self->sensor_modes);
> + if (!gst_caps_is_empty(sensor_mode)) {
> + roles.push_back(libcamera::StreamRole::Raw);
> + } else {
> + GST_WARNING_OBJECT(self, "No sensor mode matching the selection, ignoring.");
> + gst_clear_caps(&sensor_mode);
> + }
> + }
> +
> /* Generate the stream configurations, there should be one per pad. */
> state->config_ = state->cam_->generateConfiguration(roles);
> if (state->config_ == nullptr) {
> @@ -490,7 +575,7 @@ gst_libcamera_src_task_enter(GstTask *task, [[maybe_unused]] GThread *thread,
> gst_task_stop(task);
> return;
> }
> - g_assert(state->config_->size() == state->srcpads_.size());
> + g_assert(state->config_->size() == state->srcpads_.size() + (!!sensor_mode));
>
> for (gsize i = 0; i < state->srcpads_.size(); i++) {
> GstPad *srcpad = state->srcpads_[i];
> @@ -510,6 +595,12 @@ gst_libcamera_src_task_enter(GstTask *task, [[maybe_unused]] GThread *thread,
> gst_libcamera_get_framerate_from_caps(caps, element_caps);
> }
>
> + if (sensor_mode) {
> + StreamConfiguration &stream_cfg = state->config_->at(state->srcpads_.size());
> + g_assert(gst_caps_is_writable(sensor_mode));
> + gst_libcamera_configure_stream_from_caps(stream_cfg, sensor_mode);
> + }
> +
> if (flow_ret != GST_FLOW_OK)
> goto done;
>
> @@ -624,6 +715,7 @@ gst_libcamera_src_task_leave([[maybe_unused]] GstTask *task,
> g_clear_object(&self->allocator);
> g_clear_pointer(&self->flow_combiner,
> (GDestroyNotify)gst_flow_combiner_free);
> + gst_clear_caps(&self->sensor_modes);
> }
>
> static void
> @@ -659,6 +751,10 @@ gst_libcamera_src_set_property(GObject *object, guint prop_id,
> g_free(self->camera_name);
> self->camera_name = g_value_dup_string(value);
> break;
> + case PROP_SENSOR_MODE:
> + gst_clear_caps(&self->sensor_mode);
> + self->sensor_mode = GST_CAPS(g_value_dup_boxed(value));
> + break;
> default:
> G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
> break;
> @@ -676,6 +772,12 @@ gst_libcamera_src_get_property(GObject *object, guint prop_id, GValue *value,
> case PROP_CAMERA_NAME:
> g_value_set_string(value, self->camera_name);
> break;
> + case PROP_SENSOR_MODES:
> + g_value_set_boxed(value, self->sensor_modes);
> + break;
> + case PROP_SENSOR_MODE:
> + g_value_set_boxed(value, self->sensor_mode);
> + break;
> default:
> G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
> break;
> @@ -763,6 +865,7 @@ gst_libcamera_src_init(GstLibcameraSrc *self)
> /* C-style friend. */
> state->src_ = self;
> self->state = state;
> + self->sensor_mode = gst_caps_new_any();
> }
>
> static GstPad *
> @@ -844,4 +947,22 @@ gst_libcamera_src_class_init(GstLibcameraSrcClass *klass)
> | G_PARAM_READWRITE
> | G_PARAM_STATIC_STRINGS));
> g_object_class_install_property(object_class, PROP_CAMERA_NAME, spec);
> +
> + spec = g_param_spec_boxed("sensor-modes", "Sensor Modes",
> + "GstCaps representing available sensor modes.",
> + GST_TYPE_CAPS,
> + (GParamFlags)(G_PARAM_READABLE
> + | G_PARAM_STATIC_STRINGS));
> + g_object_class_install_property(object_class, PROP_SENSOR_MODES, spec);
> +
> + spec = g_param_spec_boxed("sensor-mode", "Sensor Mode",
> + "GstCaps representing selected sensor mode.",
> + GST_TYPE_CAPS,
> + (GParamFlags)(GST_PARAM_MUTABLE_READY
> + | G_PARAM_READWRITE
> + | G_PARAM_STATIC_STRINGS));
> + g_object_class_install_property(object_class, PROP_SENSOR_MODE, spec);
> +
> +
> + GstCameraControls::installProperties(object_class, PROP_LAST);
> }
More information about the libcamera-devel
mailing list