[libcamera-devel] Software gathering of image statistics for libcamera on Librem 5

[Pavel Machek]

Hi!

> > Thanks for rkisp pointer. I'm quite confused by that code, not knowing
> > much about rkisp. Does it have some special support in hardware, like
> > providing one stream of frames for application and second for
> > statistics gathering?
> 
> Yes. But when capturing RAW frames, it doesn't have the ability to
> provide statistics, so that means for RAW capture - we can't have a 3a
> loop running, and can only run with manual parameters.
> 
> 
> > For Librem 5 (and possibly PinePhone and similar), I'd like to snoop
> > on frames provided to the application, and set up exposure
> > accordingly. But I don't see any code in libcamera accessing image
> > data.
> > 
> > Closest I could get is Image class (src/apps/common/image.h) used by
> > file sinks in cam application. Would it be acceptable to move that
> > into libcamera and use it to peek at the pixels?
> 
> You can peek at pixels with the MappedFrameBuffer class.
> 
>  include/libcamera/internal/mapped_framebuffer.h
>  src/libcamera/mapped_framebuffer.cpp
> 
> If we only need 200 pixels to process, I'd be very happy to see a CPU
> based implementation that could be used on 'any' platform without an
> ISP.

Thank you. It looks like I can access all the neccessary information
from simple pipeline. I guess MappedPixels class should go somewhere
else; let me know if you have preferences for place and naming.

(There's more work to do, this just shows I have correct hooks).

Best regards,
								Pavel

diff --git a/src/libcamera/pipeline/simple/simple.cpp b/src/libcamera/pipeline/simple/simple.cpp
index a32de7f3..2980c636 100644
--- a/src/libcamera/pipeline/simple/simple.cpp
+++ b/src/libcamera/pipeline/simple/simple.cpp
@@ -27,6 +27,7 @@
 #include <libcamera/control_ids.h>
 #include <libcamera/request.h>
 #include <libcamera/stream.h>
+#include <libcamera/formats.h>
 
 #include "libcamera/internal/camera.h"
 #include "libcamera/internal/camera_sensor.h"
@@ -35,6 +36,7 @@
 #include "libcamera/internal/pipeline_handler.h"
 #include "libcamera/internal/v4l2_subdevice.h"
 #include "libcamera/internal/v4l2_videodevice.h"
+#include "libcamera/internal/mapped_framebuffer.h"
 
 #include "converter.h"
 
@@ -213,6 +215,7 @@ public:
 	int setupFormats(V4L2SubdeviceFormat *format,
 			 V4L2Subdevice::Whence whence);
 	void bufferReady(FrameBuffer *buffer);
+	void autoProcessing(Request *request);
 
 	unsigned int streamIndex(const Stream *stream) const
 	{
@@ -724,6 +727,135 @@ int SimpleCameraData::setupFormats(V4L2SubdeviceFormat *format,
 	return 0;
 }
 
+class MappedPixels {
+public:
+	unsigned char *data;
+	PixelFormat format;
+	Size size;
+	int bpp;
+	int maxval;
+  
+	MappedPixels(unsigned char *_data, const struct StreamConfiguration &_config) {
+		data = _data;
+		format = _config.pixelFormat;
+		size = _config.size;
+
+		switch (format) {
+		case formats::SGRBG8:
+			bpp = 1;
+			maxval = 255;
+			break;
+		case formats::SGRBG10:
+			bpp = 2;
+			maxval = 1023;
+			break;
+		default:
+			LOG(SimplePipeline, Error) << "Mapped pixels " << format << " is unknown format.";
+		}
+	}
+
+	int getValue(unsigned int x, unsigned int y) {
+		unsigned int v;
+		if (x >= size.width)
+			x = size.width - 1;
+		if (y >= size.height)
+			y = size.height - 1;
+		int i = (x + size.width * y) * bpp;
+		v = data[i];
+		if (bpp > 1)
+			v |= data[i+1] << 8;
+		return v;
+	}
+
+	int getMaxValue(unsigned int x, unsigned int y) {
+		int v, v2;
+
+		v = getValue(x, y);
+		v2 = getValue(x+1, y);
+		if (v2 > v)
+			v = v2;
+		v2 = getValue(x, y+1);
+		if (v2 > v)
+			v = v2;
+		v2 = getValue(x+1, y+1);
+		if (v2 > v)
+			v = v2;
+		return v;
+	}
+
+	float getMaxValueR(float x, float y) {
+		float v = getMaxValue(x * size.width, y * size.height);
+		return v/maxval;
+	}
+
+	void debugPaint(void) {
+		char map[] = "  .,:;-+=*#";
+		for (float y = 0; y < 1; y += 1/25.) {
+			for (float x = 0; x < 1; x += 1/80.) {
+				float v = getMaxValueR(x, y);
+				printf("%c", map[ int (v * (sizeof(map) - 2)) ]);
+			}
+			printf("\n");
+		}
+	}	
+};
+
+void SimpleCameraData::autoProcessing(Request *request)
+{
+	ControlList ctrls = sensor_->getControls({ V4L2_CID_EXPOSURE, V4L2_CID_ANALOGUE_GAIN });
+	static int exposure = 64;
+	static int gain = 64;
+
+	/* good place to add my processing ? */
+        for (auto [stream, buffer] : request->buffers()) {
+		MappedFrameBuffer mappedBuffer(buffer, MappedFrameBuffer::MapFlag::Read);
+		const std::vector<Span<uint8_t>> &planes = mappedBuffer.planes();
+		unsigned char *img = planes[0].data();
+		const struct StreamConfiguration &config = stream->configuration();
+		MappedPixels pixels(img, config);
+
+ 		LOG(SimplePipeline, Error) << config.pixelFormat << " " << config.size;
+
+		pixels.debugPaint();
+
+		int bright = 0, too_bright = 0, total = 0;
+		int adjust = 0;
+
+		for (float y = 0; y < 1; y += 1/30.) {
+			for (float x = 0; x < 1; x += 1/40.) {
+				float v = pixels.getMaxValueR(x, y);
+
+				total++;
+				if (v > 240./255)
+					too_bright++;
+				if (v > 200./255)
+					bright++;
+			}
+		}
+
+		if ((bright / (float) total) < 0.01)
+			adjust = +1;
+		if ((too_bright / (float) total) > 0.08)
+			adjust = -1;
+
+		exposure += adjust;
+		gain += adjust;
+		printf("image data -- %d %d -- %d -- exp %d\n", 
+			(int) img[0], (int) img[1], adjust, exposure);
+
+		ctrls.set(V4L2_CID_EXPOSURE, exposure);
+		ctrls.set(V4L2_CID_ANALOGUE_GAIN, gain);
+	}
+#if 0
+	const ControlInfoMap &infoMap = controls();
+
+	if (infoMap.find(V4L2_CID_BRIGHTNESS) != infoMap.end()) {
+		//const ControlInfo &brightness = infoMap.find(V4L2_CID_BRIGHTNESS)->second;
+	}
+#endif
+
+}
+
 void SimpleCameraData::bufferReady(FrameBuffer *buffer)
 {
 	SimplePipelineHandler *pipe = SimpleCameraData::pipe();
@@ -739,6 +871,7 @@ void SimpleCameraData::bufferReady(FrameBuffer *buffer)
 			Request *request = buffer->request();
 			pipe->completeBuffer(request, buffer);
 			pipe->completeRequest(request);
+
 			return;
 		}
 
@@ -761,8 +894,9 @@ void SimpleCameraData::bufferReady(FrameBuffer *buffer)
 		}
 		converterQueue_.pop();
 
-		if (request)
+		if (request) {
 			pipe->completeRequest(request);
+		}
 		return;
 	}
 
@@ -808,6 +942,7 @@ void SimpleCameraData::bufferReady(FrameBuffer *buffer)
 
 	/* Otherwise simply complete the request. */
 	pipe->completeBuffer(request, buffer);
+	autoProcessing(request);
 	pipe->completeRequest(request);
 }
 
@@ -823,8 +958,10 @@ void SimpleCameraData::converterOutputDone(FrameBuffer *buffer)
 
 	/* Complete the buffer and the request. */
 	Request *request = buffer->request();
-	if (pipe->completeBuffer(request, buffer))
+	if (pipe->completeBuffer(request, buffer)) {
+		autoProcessing(request);
 		pipe->completeRequest(request);
+	}
 }
 
 /* Retrieve all source pads connected to a sink pad through active routes. */


-- 
People of Russia, stop Putin before his war on Ukraine escalates.
-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 195 bytes
Desc: not available
URL: <https://lists.libcamera.org/pipermail/libcamera-devel/attachments/20221210/6a49f8a0/attachment.sig>