[libcamera-devel] Software gathering of image statistics for libcamera on Librem 5

[Laurent Pinchart]

Hi Pavel,

On Sat, Dec 10, 2022 at 10:09:20PM +0100, Pavel Machek via libcamera-devel wrote:
> Hi!
> 
> > > Thanks for rkisp pointer. I'm quite confused by that code, not knowing
> > > much about rkisp. Does it have some special support in hardware, like
> > > providing one stream of frames for application and second for
> > > statistics gathering?
> > 
> > Yes. But when capturing RAW frames, it doesn't have the ability to
> > provide statistics, so that means for RAW capture - we can't have a 3a
> > loop running, and can only run with manual parameters.
> > 
> > > For Librem 5 (and possibly PinePhone and similar), I'd like to snoop
> > > on frames provided to the application, and set up exposure
> > > accordingly. But I don't see any code in libcamera accessing image
> > > data.
> > > 
> > > Closest I could get is Image class (src/apps/common/image.h) used by
> > > file sinks in cam application. Would it be acceptable to move that
> > > into libcamera and use it to peek at the pixels?
> > 
> > You can peek at pixels with the MappedFrameBuffer class.
> > 
> >  include/libcamera/internal/mapped_framebuffer.h
> >  src/libcamera/mapped_framebuffer.cpp
> > 
> > If we only need 200 pixels to process, I'd be very happy to see a CPU
> > based implementation that could be used on 'any' platform without an
> > ISP.
> 
> Thank you. It looks like I can access all the neccessary information
> from simple pipeline. I guess MappedPixels class should go somewhere
> else; let me know if you have preferences for place and naming.

Please see my other reply in this mail thread. I would like to see a
base abstract class to model a software ISP.

> (There's more work to do, this just shows I have correct hooks).
> 
> Best regards,
> 								Pavel
> 
> diff --git a/src/libcamera/pipeline/simple/simple.cpp b/src/libcamera/pipeline/simple/simple.cpp
> index a32de7f3..2980c636 100644
> --- a/src/libcamera/pipeline/simple/simple.cpp
> +++ b/src/libcamera/pipeline/simple/simple.cpp
> @@ -27,6 +27,7 @@
>  #include <libcamera/control_ids.h>
>  #include <libcamera/request.h>
>  #include <libcamera/stream.h>
> +#include <libcamera/formats.h>
>  
>  #include "libcamera/internal/camera.h"
>  #include "libcamera/internal/camera_sensor.h"
> @@ -35,6 +36,7 @@
>  #include "libcamera/internal/pipeline_handler.h"
>  #include "libcamera/internal/v4l2_subdevice.h"
>  #include "libcamera/internal/v4l2_videodevice.h"
> +#include "libcamera/internal/mapped_framebuffer.h"
>  
>  #include "converter.h"
>  
> @@ -213,6 +215,7 @@ public:
>  	int setupFormats(V4L2SubdeviceFormat *format,
>  			 V4L2Subdevice::Whence whence);
>  	void bufferReady(FrameBuffer *buffer);
> +	void autoProcessing(Request *request);
>  
>  	unsigned int streamIndex(const Stream *stream) const
>  	{
> @@ -724,6 +727,135 @@ int SimpleCameraData::setupFormats(V4L2SubdeviceFormat *format,
>  	return 0;
>  }
>  
> +class MappedPixels {
> +public:
> +	unsigned char *data;
> +	PixelFormat format;
> +	Size size;
> +	int bpp;
> +	int maxval;
> +  
> +	MappedPixels(unsigned char *_data, const struct StreamConfiguration &_config) {
> +		data = _data;
> +		format = _config.pixelFormat;
> +		size = _config.size;
> +
> +		switch (format) {
> +		case formats::SGRBG8:
> +			bpp = 1;
> +			maxval = 255;
> +			break;
> +		case formats::SGRBG10:
> +			bpp = 2;
> +			maxval = 1023;
> +			break;
> +		default:
> +			LOG(SimplePipeline, Error) << "Mapped pixels " << format << " is unknown format.";
> +		}
> +	}
> +
> +	int getValue(unsigned int x, unsigned int y) {
> +		unsigned int v;
> +		if (x >= size.width)
> +			x = size.width - 1;
> +		if (y >= size.height)
> +			y = size.height - 1;
> +		int i = (x + size.width * y) * bpp;
> +		v = data[i];
> +		if (bpp > 1)
> +			v |= data[i+1] << 8;
> +		return v;
> +	}
> +
> +	int getMaxValue(unsigned int x, unsigned int y) {
> +		int v, v2;
> +
> +		v = getValue(x, y);
> +		v2 = getValue(x+1, y);
> +		if (v2 > v)
> +			v = v2;
> +		v2 = getValue(x, y+1);
> +		if (v2 > v)
> +			v = v2;
> +		v2 = getValue(x+1, y+1);
> +		if (v2 > v)
> +			v = v2;
> +		return v;
> +	}
> +
> +	float getMaxValueR(float x, float y) {
> +		float v = getMaxValue(x * size.width, y * size.height);
> +		return v/maxval;
> +	}
> +
> +	void debugPaint(void) {
> +		char map[] = "  .,:;-+=*#";
> +		for (float y = 0; y < 1; y += 1/25.) {
> +			for (float x = 0; x < 1; x += 1/80.) {
> +				float v = getMaxValueR(x, y);
> +				printf("%c", map[ int (v * (sizeof(map) - 2)) ]);
> +			}
> +			printf("\n");
> +		}
> +	}	
> +};
> +
> +void SimpleCameraData::autoProcessing(Request *request)
> +{
> +	ControlList ctrls = sensor_->getControls({ V4L2_CID_EXPOSURE, V4L2_CID_ANALOGUE_GAIN });
> +	static int exposure = 64;
> +	static int gain = 64;
> +
> +	/* good place to add my processing ? */
> +        for (auto [stream, buffer] : request->buffers()) {
> +		MappedFrameBuffer mappedBuffer(buffer, MappedFrameBuffer::MapFlag::Read);
> +		const std::vector<Span<uint8_t>> &planes = mappedBuffer.planes();
> +		unsigned char *img = planes[0].data();
> +		const struct StreamConfiguration &config = stream->configuration();
> +		MappedPixels pixels(img, config);
> +
> + 		LOG(SimplePipeline, Error) << config.pixelFormat << " " << config.size;
> +
> +		pixels.debugPaint();
> +
> +		int bright = 0, too_bright = 0, total = 0;
> +		int adjust = 0;
> +
> +		for (float y = 0; y < 1; y += 1/30.) {
> +			for (float x = 0; x < 1; x += 1/40.) {
> +				float v = pixels.getMaxValueR(x, y);
> +
> +				total++;
> +				if (v > 240./255)
> +					too_bright++;
> +				if (v > 200./255)
> +					bright++;
> +			}
> +		}
> +
> +		if ((bright / (float) total) < 0.01)
> +			adjust = +1;
> +		if ((too_bright / (float) total) > 0.08)
> +			adjust = -1;
> +
> +		exposure += adjust;
> +		gain += adjust;
> +		printf("image data -- %d %d -- %d -- exp %d\n", 
> +			(int) img[0], (int) img[1], adjust, exposure);
> +
> +		ctrls.set(V4L2_CID_EXPOSURE, exposure);
> +		ctrls.set(V4L2_CID_ANALOGUE_GAIN, gain);
> +	}
> +#if 0
> +	const ControlInfoMap &infoMap = controls();
> +
> +	if (infoMap.find(V4L2_CID_BRIGHTNESS) != infoMap.end()) {
> +		//const ControlInfo &brightness = infoMap.find(V4L2_CID_BRIGHTNESS)->second;
> +	}
> +#endif
> +
> +}
> +
>  void SimpleCameraData::bufferReady(FrameBuffer *buffer)
>  {
>  	SimplePipelineHandler *pipe = SimpleCameraData::pipe();
> @@ -739,6 +871,7 @@ void SimpleCameraData::bufferReady(FrameBuffer *buffer)
>  			Request *request = buffer->request();
>  			pipe->completeBuffer(request, buffer);
>  			pipe->completeRequest(request);
> +
>  			return;
>  		}
>  
> @@ -761,8 +894,9 @@ void SimpleCameraData::bufferReady(FrameBuffer *buffer)
>  		}
>  		converterQueue_.pop();
>  
> -		if (request)
> +		if (request) {
>  			pipe->completeRequest(request);
> +		}
>  		return;
>  	}
>  
> @@ -808,6 +942,7 @@ void SimpleCameraData::bufferReady(FrameBuffer *buffer)
>  
>  	/* Otherwise simply complete the request. */
>  	pipe->completeBuffer(request, buffer);
> +	autoProcessing(request);
>  	pipe->completeRequest(request);
>  }
>  
> @@ -823,8 +958,10 @@ void SimpleCameraData::converterOutputDone(FrameBuffer *buffer)
>  
>  	/* Complete the buffer and the request. */
>  	Request *request = buffer->request();
> -	if (pipe->completeBuffer(request, buffer))
> +	if (pipe->completeBuffer(request, buffer)) {
> +		autoProcessing(request);
>  		pipe->completeRequest(request);
> +	}
>  }
>  
>  /* Retrieve all source pads connected to a sink pad through active routes. */

-- 
Regards,

Laurent Pinchart