[libcamera-devel] [PATCH 8/8] gstreamer: Use Sensor sequence numbers and detect frame drop

[Kieran Bingham]

The stream buffer sequence numbers might produce sequential
monotonic sequence numbers from an ISP producing a frame for every
input.

This however, doesn't capture pipeline stalls that cause us to miss or
drop frames from the sensor.

Use the SensorSequence metadata to report sequence information, and
report to the application if a frame drop is detected.

Signed-off-by: Kieran Bingham <kieran.bingham at ideasonboard.com>

---
RFC: This one likely needs better plumbing into gstreamer events.
Is there anything we should specifically signal to gst ?

Is there a better way to report the errors?
---
 src/gstreamer/gstlibcamerasrc.cpp | 42 ++++++++++++++++++++++++++-----
 1 file changed, 36 insertions(+), 6 deletions(-)

diff --git a/src/gstreamer/gstlibcamerasrc.cpp b/src/gstreamer/gstlibcamerasrc.cpp
index c7292f66b17b..8653127f0d85 100644
--- a/src/gstreamer/gstlibcamerasrc.cpp
+++ b/src/gstreamer/gstlibcamerasrc.cpp
@@ -33,6 +33,7 @@
 #include <vector>
 
 #include <libcamera/camera.h>
+#include <libcamera/control_ids.h>
 #include <libcamera/camera_manager.h>
 
 #include <gst/base/base.h>
@@ -111,6 +112,8 @@ struct GstLibcameraSrcState {
 	std::queue<std::unique_ptr<RequestWrap>> requests_;
 	guint group_id_;
 
+	guint64 sequence_;
+
 	void requestCompleted(Request *request);
 };
 
@@ -165,13 +168,40 @@ GstLibcameraSrcState::requestCompleted(Request *request)
 		return;
 	}
 
+	/* Extract request metadata */
+	uint64_t sequence = 0;
+	uint64_t timestamp = 0;
+
+	for (const auto &ctrl : request->metadata()) {
+		const int id = ctrl.first;
+		const ControlValue &value = ctrl.second;
+
+		if (id == controls::SensorSequence) {
+			/* Handle basic frame drop detection and reporting. */
+			sequence = value.get<int64_t>();
+			if (sequence_ == 0)
+				sequence_ = sequence - 1;
+			unsigned int drops = sequence - sequence_ - 1;
+			if (drops)
+				GST_ELEMENT_WARNING(src_, RESOURCE, BUSY,
+						    ("Camera '%s' dropped %u frames.", cam_->id().c_str(), drops),
+						    ("libcamera::Request.metadata() reports %u dropped frames at sequence %lu", drops, sequence));
+
+			sequence_ = sequence;
+		}
+
+		if (id == controls::SensorTimestamp) {
+			timestamp = value.get<int64_t>();
+		}
+
+		/* \todo Handle all/other metadata types here. */
+	}
+
 	GstBuffer *buffer;
 	for (GstPad *srcpad : srcpads_) {
 		Stream *stream = gst_libcamera_pad_get_stream(srcpad);
 		buffer = wrap->detachBuffer(stream);
 
-		FrameBuffer *fb = gst_libcamera_buffer_get_frame_buffer(buffer);
-
 		if (GST_ELEMENT_CLOCK(src_)) {
 			GstClockTime gst_base_time = GST_ELEMENT(src_)->base_time;
 			GstClockTime gst_now = gst_clock_get_time(GST_ELEMENT_CLOCK(src_));
@@ -180,14 +210,14 @@ GstLibcameraSrcState::requestCompleted(Request *request)
 
 			/* Deduced from: sys_now - sys_base_time == gst_now - gst_base_time */
 			GstClockTime sys_base_time = sys_now - (gst_now - gst_base_time);
-			GST_BUFFER_PTS(buffer) = fb->metadata().timestamp - sys_base_time;
-			gst_libcamera_pad_set_latency(srcpad, sys_now - fb->metadata().timestamp);
+			GST_BUFFER_PTS(buffer) = timestamp - sys_base_time;
+			gst_libcamera_pad_set_latency(srcpad, sys_now - timestamp);
 		} else {
 			GST_BUFFER_PTS(buffer) = 0;
 		}
 
-		GST_BUFFER_OFFSET(buffer) = fb->metadata().sequence;
-		GST_BUFFER_OFFSET_END(buffer) = fb->metadata().sequence;
+		GST_BUFFER_OFFSET(buffer) = sequence;
+		GST_BUFFER_OFFSET_END(buffer) = sequence;
 
 		gst_libcamera_pad_queue_buffer(srcpad, buffer);
 	}
-- 
2.30.2