[libcamera-devel] [PATCH v4 11/11] libcamera: ipu3: Share parameter and statistic buffers with IPA
Laurent Pinchart
laurent.pinchart at ideasonboard.com
Fri Feb 5 00:47:03 CET 2021
Hi Niklas,
Thank you for the patch.
On Fri, Feb 05, 2021 at 12:26:13AM +0100, Niklas Söderlund wrote:
> Use the IPU3Frames helper to share parameters and statistics buffers
> with the IPA. Track which parameter and statistic buffer is used for
> which request and make sure the parameter buffers is filled in by the
> IPA before it's needed and that the statistic buffer is consumed and
> meta data generated before completing the request.
>
> With this change the IPU3 pipeline is prepared to fully operate with an
> IPA component.
>
> Signed-off-by: Niklas Söderlund <niklas.soderlund at ragnatech.se>
Reviewed-by: Laurent Pinchart <laurent.pinchart at ideasonboard.com>
> ---
> * Changes since v1
> - Update commit message.
> - s/frameInfo_/frameInfos_/
> - Refactor away isComplete variable.
>
> * Changes since v2
> - Emedd the IPU3Frames instance instead of allocating it.
> - Use -ENOBUFS instead of _ENOENT in queueRequestDevice().
> - Keep isComplete in cio2BufferReady()
> - Rework the queue logic.
>
> * Changes since v3
> - Fix error path in queueRequestDevice().
> - Simplify error logic in cio2BufferReady().
> - Handle canceled stat buffers correctly.
> - Hand request parameters to IPA separatly from asking it to fill the
> parameters buffer.
> ---
> src/libcamera/pipeline/ipu3/ipu3.cpp | 158 ++++++++++++++++++++-------
> 1 file changed, 120 insertions(+), 38 deletions(-)
>
> diff --git a/src/libcamera/pipeline/ipu3/ipu3.cpp b/src/libcamera/pipeline/ipu3/ipu3.cpp
> index 092db6389833a481..c3763507c40a5d53 100644
> --- a/src/libcamera/pipeline/ipu3/ipu3.cpp
> +++ b/src/libcamera/pipeline/ipu3/ipu3.cpp
> @@ -29,6 +29,7 @@
> #include "libcamera/internal/v4l2_controls.h"
>
> #include "cio2.h"
> +#include "frames.h"
> #include "imgu.h"
>
> namespace libcamera {
> @@ -61,6 +62,8 @@ public:
>
> void imguOutputBufferReady(FrameBuffer *buffer);
> void cio2BufferReady(FrameBuffer *buffer);
> + void paramBufferReady(FrameBuffer *buffer);
> + void statBufferReady(FrameBuffer *buffer);
>
> CIO2Device cio2_;
> ImgUDevice *imgu_;
> @@ -71,6 +74,7 @@ public:
>
> uint32_t exposureTime_;
> std::unique_ptr<DelayedControls> delayedCtrls_;
> + IPU3Frames frameInfos_;
>
> private:
> void queueFrameAction(unsigned int id, const IPAOperationData &op);
> @@ -609,6 +613,8 @@ int PipelineHandlerIPU3::allocateBuffers(Camera *camera)
>
> data->ipa_->mapBuffers(ipaBuffers_);
>
> + data->frameInfos_.init(imgu->paramBuffers_, imgu->statBuffers_);
> +
> return 0;
> }
>
> @@ -616,6 +622,8 @@ int PipelineHandlerIPU3::freeBuffers(Camera *camera)
> {
> IPU3CameraData *data = cameraData(camera);
>
> + data->frameInfos_.clear();
> +
> std::vector<unsigned int> ids;
> for (IPABuffer &ipabuf : ipaBuffers_)
> ids.push_back(ipabuf.id);
> @@ -713,7 +721,10 @@ void PipelineHandlerIPU3::stop(Camera *camera)
> int PipelineHandlerIPU3::queueRequestDevice(Camera *camera, Request *request)
> {
> IPU3CameraData *data = cameraData(camera);
> - int error = 0;
> +
> + IPU3Frames::Info *info = data->frameInfos_.create(request);
> + if (!info)
> + return -ENOBUFS;
>
> /*
> * Queue a buffer on the CIO2, using the raw stream buffer provided in
> @@ -721,27 +732,20 @@ int PipelineHandlerIPU3::queueRequestDevice(Camera *camera, Request *request)
> */
> FrameBuffer *reqRawBuffer = request->findBuffer(&data->rawStream_);
> FrameBuffer *rawBuffer = data->cio2_.queueBuffer(request, reqRawBuffer);
> - if (!rawBuffer)
> + if (!rawBuffer) {
> + data->frameInfos_.remove(info);
> return -ENOMEM;
> -
> - /* Queue all buffers from the request aimed for the ImgU. */
> - for (auto it : request->buffers()) {
> - const Stream *stream = it.first;
> - FrameBuffer *buffer = it.second;
> - int ret;
> -
> - if (stream == &data->outStream_)
> - ret = data->imgu_->output_->queueBuffer(buffer);
> - else if (stream == &data->vfStream_)
> - ret = data->imgu_->viewfinder_->queueBuffer(buffer);
> - else
> - continue;
> -
> - if (ret < 0)
> - error = ret;
> }
>
> - return error;
> + info->rawBuffer = rawBuffer;
> +
> + IPAOperationData op;
> + op.operation = IPU3_IPA_EVENT_PROCESS_CONTROLS;
> + op.data = { info->id };
> + op.controls = { request->controls() };
> + data->ipa_->processEvent(op);
> +
> + return 0;
> }
>
> bool PipelineHandlerIPU3::match(DeviceEnumerator *enumerator)
> @@ -1007,6 +1011,10 @@ int PipelineHandlerIPU3::registerCameras()
> &IPU3CameraData::imguOutputBufferReady);
> data->imgu_->viewfinder_->bufferReady.connect(data.get(),
> &IPU3CameraData::imguOutputBufferReady);
> + data->imgu_->param_->bufferReady.connect(data.get(),
> + &IPU3CameraData::paramBufferReady);
> + data->imgu_->stat_->bufferReady.connect(data.get(),
> + &IPU3CameraData::statBufferReady);
>
> /* Create and register the Camera instance. */
> std::string cameraId = cio2->sensor()->id();
> @@ -1039,7 +1047,7 @@ int IPU3CameraData::loadIPA()
> return 0;
> }
>
> -void IPU3CameraData::queueFrameAction([[maybe_unused]] unsigned int id,
> +void IPU3CameraData::queueFrameAction(unsigned int id,
> const IPAOperationData &action)
> {
> switch (action.operation) {
> @@ -1048,6 +1056,41 @@ void IPU3CameraData::queueFrameAction([[maybe_unused]] unsigned int id,
> delayedCtrls_->push(controls);
> break;
> }
> + case IPU3_IPA_ACTION_PARAM_FILLED: {
> + IPU3Frames::Info *info = frameInfos_.find(id);
> + if (!info)
> + break;
> +
> + /* Queue all buffers from the request aimed for the ImgU. */
> + for (auto it : info->request->buffers()) {
> + const Stream *stream = it.first;
> + FrameBuffer *outbuffer = it.second;
> +
> + if (stream == &outStream_)
> + imgu_->output_->queueBuffer(outbuffer);
> + else if (stream == &vfStream_)
> + imgu_->viewfinder_->queueBuffer(outbuffer);
> + }
> +
> + imgu_->param_->queueBuffer(info->paramBuffer);
> + imgu_->stat_->queueBuffer(info->statBuffer);
> + imgu_->input_->queueBuffer(info->rawBuffer);
> +
> + break;
> + }
> + case IPU3_IPA_ACTION_METADATA_READY: {
> + IPU3Frames::Info *info = frameInfos_.find(id);
> + if (!info)
> + break;
> +
> + Request *request = info->request;
> + request->metadata() = action.controls[0];
> + info->metadataProcessed = true;
> + if (frameInfos_.tryComplete(info))
> + pipe_->completeRequest(request);
> +
> + break;
> + }
> default:
> LOG(IPU3, Error) << "Unknown action " << action.operation;
> break;
> @@ -1068,11 +1111,12 @@ void IPU3CameraData::imguOutputBufferReady(FrameBuffer *buffer)
> {
> Request *request = buffer->request();
>
> - if (!pipe_->completeBuffer(request, buffer))
> - /* Request not completed yet, return here. */
> + pipe_->completeBuffer(request, buffer);
> +
> + IPU3Frames::Info *info = frameInfos_.find(buffer);
> + if (!info)
> return;
>
> - /* Mark the request as complete. */
> request->metadata().set(controls::draft::PipelineDepth, 3);
> /* \todo Move the ExposureTime control to the IPA. */
> request->metadata().set(controls::ExposureTime, exposureTime_);
> @@ -1081,7 +1125,12 @@ void IPU3CameraData::imguOutputBufferReady(FrameBuffer *buffer)
> Rectangle cropRegion = request->controls().get(controls::ScalerCrop);
> request->metadata().set(controls::ScalerCrop, cropRegion);
> }
> - pipe_->completeRequest(request);
> +
> + if (buffer->metadata().status == FrameMetadata::FrameCancelled)
> + info->metadataProcessed = true;
> +
> + if (frameInfos_.tryComplete(info))
> + pipe_->completeRequest(request);
> }
>
> /**
> @@ -1093,26 +1142,59 @@ void IPU3CameraData::imguOutputBufferReady(FrameBuffer *buffer)
> */
> void IPU3CameraData::cio2BufferReady(FrameBuffer *buffer)
> {
> - /* \todo Handle buffer failures when state is set to BufferError. */
> - if (buffer->metadata().status == FrameMetadata::FrameCancelled)
> + IPU3Frames::Info *info = frameInfos_.find(buffer);
> + if (!info)
> return;
>
> Request *request = buffer->request();
>
> - /*
> - * If the request contains a buffer for the RAW stream only, complete it
> - * now as there's no need for ImgU processing.
> - */
> - if (request->findBuffer(&rawStream_)) {
> - bool isComplete = pipe_->completeBuffer(request, buffer);
> - if (isComplete) {
> - request->metadata().set(controls::draft::PipelineDepth, 2);
> - pipe_->completeRequest(request);
> - return;
> - }
> + /* If the buffer is cancelled force a complete of the whole request. */
> + if (buffer->metadata().status == FrameMetadata::FrameCancelled) {
> + for (auto it : request->buffers())
> + pipe_->completeBuffer(request, it.second);
> +
> + frameInfos_.remove(info);
> + pipe_->completeRequest(request);
> + return;
> + }
> +
> + if (request->findBuffer(&rawStream_))
> + pipe_->completeBuffer(request, buffer);
> +
> + IPAOperationData op;
> + op.operation = IPU3_IPA_EVENT_FILL_PARAMS;
> + op.data = { info->id, info->paramBuffer->cookie() };
> + ipa_->processEvent(op);
> +}
> +
> +void IPU3CameraData::paramBufferReady(FrameBuffer *buffer)
> +{
> + IPU3Frames::Info *info = frameInfos_.find(buffer);
> + if (!info)
> + return;
> +
> + info->paramDequeued = true;
> + if (frameInfos_.tryComplete(info))
> + pipe_->completeRequest(buffer->request());
> +}
> +
> +void IPU3CameraData::statBufferReady(FrameBuffer *buffer)
> +{
> + IPU3Frames::Info *info = frameInfos_.find(buffer);
> + if (!info)
> + return;
> +
> + if (buffer->metadata().status == FrameMetadata::FrameCancelled) {
> + info->metadataProcessed = true;
> + if (frameInfos_.tryComplete(info))
> + pipe_->completeRequest(buffer->request());
> + return;
> }
>
> - imgu_->input_->queueBuffer(buffer);
> + IPAOperationData op;
> + op.operation = IPU3_IPA_EVENT_STAT_READY;
> + op.data = { info->id, info->statBuffer->cookie() };
> + ipa_->processEvent(op);
> }
>
> REGISTER_PIPELINE_HANDLER(PipelineHandlerIPU3)
--
Regards,
Laurent Pinchart
More information about the libcamera-devel
mailing list