[libcamera-devel] [RFC PATCH v2 1/4] pipeline: isp: The software ISP-based pipeline handler
paul.elder at ideasonboard.com
paul.elder at ideasonboard.com
Wed Aug 11 12:03:39 CEST 2021
Hi Siyuan,
On Wed, Aug 11, 2021 at 07:12:55AM +0100, Siyuan Fan wrote:
> From: Fan Siyuan <siyuan.fan at foxmail.com>
>
> Changes in V2:
> -fix the raw and rgb data flow based queue model in pipeline handler
> -move buffer alloc and thread to ISP class
> -Fill metadata information in dstBuffer
>
> Signed-off-by: Fan Siyuan <siyuan.fan at foxmail.com>
> ---
> src/libcamera/pipeline/isp/isp.cpp | 306 +++++++++++++++++++++++++++++
> 1 file changed, 306 insertions(+)
> create mode 100644 src/libcamera/pipeline/isp/isp.cpp
>
> diff --git a/src/libcamera/pipeline/isp/isp.cpp b/src/libcamera/pipeline/isp/isp.cpp
> new file mode 100644
> index 00000000..c6b7808c
> --- /dev/null
> +++ b/src/libcamera/pipeline/isp/isp.cpp
> @@ -0,0 +1,306 @@
> +/* SPDX-License-Identifier: LGPL-2.1-or-later */
> +/*
> + * Copyright (C) 2021, Siyuan Fan <siyuan.fan at foxmail.com>
> + *
> + * isp.cpp - The software ISP-based pipeline handler
> + */
> +
> +#include "../../swisp/isp.h"
> +
> +#include <math.h>
> +#include <queue>
> +#include <stdlib.h>
> +#include <sys/mman.h>
> +#include <unistd.h>
> +
> +#include <libcamera/camera.h>
> +#include <libcamera/control_ids.h>
> +#include <libcamera/controls.h>
> +#include <libcamera/formats.h>
> +
> +#include "libcamera/internal/device_enumerator.h"
> +#include "libcamera/internal/media_device.h"
> +#include "libcamera/internal/pipeline_handler.h"
> +#include "libcamera/internal/v4l2_videodevice.h"
> +
> +namespace libcamera {
> +
> +LOG_DEFINE_CATEGORY(ISP)
> +
> +class ISPCameraData : public CameraData
> +{
> +public:
> + ISPCameraData(PipelineHandler *pipe, MediaDevice *media)
> + : CameraData(pipe), media_(media), video_(nullptr)
> + {
> + }
> +
> + ~ISPCameraData()
> + {
> + delete video_;
> + }
> +
> + int init();
> + void bufferReady(FrameBuffer *buffer);
> + void ISPCompleted(FrameBuffer *buffer);
> +
> + Stream stream_;
> + CPU_ISP isp_;
> + int width_;
> + int height_;
> +
> + std::vector<std::unique_ptr<FrameBuffer>> rawBuffers_;
> + std::vector<FrameBuffer *> rawQueueBuffers_;
> + std::vector<FrameBuffer *> rgbQueueBuffers_;
These are queues, so they should be std::queue
Also in english we would say rawBufferQueue.
> +
> + MediaDevice *media_;
> + V4L2VideoDevice *video_;
> +};
> +
> +
> +class ISPCameraConfiguration : public CameraConfiguration
> +{
> +public:
> + ISPCameraConfiguration();
> +
> + Status validate() override;
> +};
> +
> +class PipelineHandlerISP : public PipelineHandler
> +{
> +public:
> + PipelineHandlerISP(CameraManager *manager);
> +
> + CameraConfiguration *generateConfiguration(Camera *camera,
> + const StreamRoles &roles) override;
> + int configure(Camera *camera, CameraConfiguration *config) override;
> +
> + int exportFrameBuffers(Camera *camera, Stream *stream,
> + std::vector<std::unique_ptr<FrameBuffer>> *buffers) override;
> +
> + int start(Camera *camera, const ControlList *controls) override;
> + void stop(Camera *camera) override;
> +
> + int queueRequestDevice(Camera *camera, Request *request) override;
> +
> + bool match(DeviceEnumerator *enumerator) override;
> +
> +private:
> + ISPCameraData *cameraData(const Camera *camera)
> + {
> + return static_cast<ISPCameraData *>(
> + PipelineHandler::cameraData(camera));
> + }
> +};
> +
> +ISPCameraConfiguration::ISPCameraConfiguration()
> + : CameraConfiguration()
> +{
> +}
> +
> +CameraConfiguration::Status ISPCameraConfiguration::validate()
> +{
> + Status status = Valid;
> +
> + return status;
> +}
> +
> +PipelineHandlerISP::PipelineHandlerISP(CameraManager *manager)
> + : PipelineHandler(manager)
> +{
> +}
> +
> +CameraConfiguration *PipelineHandlerISP::generateConfiguration(Camera *camera,
> + const StreamRoles &roles)
> +{
> + ISPCameraData *data = cameraData(camera);
> + CameraConfiguration *config = new ISPCameraConfiguration();
> +
> + if (roles.empty())
> + return config;
> +
> + std::map<V4L2PixelFormat, std::vector<SizeRange>> v4l2Formats =
> + data->video_->formats();
Similar to this, I think it would be good for the ISP to report what
formats it supports.
> + std::map<PixelFormat, std::vector<SizeRange>> deviceFormats;
> + std::transform(v4l2Formats.begin(), v4l2Formats.end(),
> + std::inserter(deviceFormats, deviceFormats.begin()),
> + [&](const decltype(v4l2Formats)::value_type &format) {
> + return decltype(deviceFormats)::value_type{
> + format.first.toPixelFormat(),
> + format.second
> + };
> + });
> +
> + StreamFormats formats(deviceFormats);
> + StreamConfiguration cfg(formats);
> +
> + cfg.pixelFormat = formats::RGB888;
> + cfg.size = { 640, 480 };
> + cfg.bufferCount = 4;
> +
> + config->addConfiguration(cfg);
> +
> + config->validate();
> +
> + return config;
> +}
> +
> +int PipelineHandlerISP::configure(Camera *camera, CameraConfiguration *config)
> +{
> + ISPCameraData *data = cameraData(camera);
> + StreamConfiguration &cfg = config->at(0);
> +
> + V4L2VideoDevice::Formats fmts = data->video_->formats();
> + V4L2PixelFormat v4l2Format = fmts.begin()->first;
> +
> + V4L2DeviceFormat format = {};
> + format.fourcc = v4l2Format;
> + format.size = cfg.size;
> +
> + data->width_ = format.size.width;
> + data->height_ = format.size.height;
> +
> + int ret = data->video_->setFormat(&format);
> + if (ret)
> + return ret;
The ISP should have a configure() function of some sort as well.
> +
> + cfg.setStream(&data->stream_);
> + cfg.stride = format.planes[0].bpl;
> +
> + return 0;
> +}
> +
> +int PipelineHandlerISP::exportFrameBuffers(Camera *camera, Stream *stream,
> + std::vector<std::unique_ptr<FrameBuffer>> *buffers)
> +{
> + unsigned int count = stream->configuration().bufferCount;
> + ISPCameraData *data = cameraData(camera);
> +
> + count = data->isp_.exportBuffers(buffers, count, data->width_, data->height_);
> +
> + return count;
> +
> +}
> +
> +int PipelineHandlerISP::start(Camera *camera, [[maybe_unused]] const ControlList *controls)
> +{
> + ISPCameraData *data = cameraData(camera);
> + unsigned int count = data->stream_.configuration().bufferCount;
> +
> + int ret = data->video_->allocateBuffers(count, &data->rawBuffers_);
> + if (ret < 0) {
> + LOG(ISP, Error) << strerror(-ret);
> + return ret;
> + }
> +
> + for (unsigned int i = 0; i < count; i++) {
> + data->rawQueueBuffers_.push_back(data->rawBuffers_[i].get());
> + }
> +
> +
> + ret = data->video_->streamOn();
> + if (ret < 0) {
> + data->video_->releaseBuffers();
> + return ret;
> + }
> +
> + data->isp_.startThreadISP();
> +
> + return 0;
> +}
> +
> +void PipelineHandlerISP::stop(Camera *camera)
> +{
> + ISPCameraData *data = cameraData(camera);
> +
> + if (!(data->rawBuffers_.empty())) {
> + data->rawBuffers_.clear();
> + }
> +
> + data->isp_.stopThreadISP();
> +
> + data->video_->streamOff();
> + data->video_->releaseBuffers();
> +}
> +
> +int PipelineHandlerISP::queueRequestDevice(Camera *camera, Request *request)
> +{
> + ISPCameraData *data = cameraData(camera);
> + FrameBuffer *rgbBuffer = request->findBuffer(&data->stream_);
> + data->rgbQueueBuffers_.push_back(rgbBuffer);
This should go after the error check. If there's no buffer then you
should't queue it.
> +
> + if (!rgbBuffer) {
> + LOG(ISP, Error) << "Attempt to queue request with invalid stream";
> + return -ENOENT;
> + }
> +
FrameBuffer *buffer = data->rawQueueBuffers_.front();
> + int ret = data->video_->queueBuffer(data->rawQueueBuffers_[0]);
data->video_->queueBuffer(buffer);
> + if (ret < 0)
> + return ret;
> + FrameBuffer *temp = data->rawQueueBuffers_[0];
> + data->rawQueueBuffers_.erase(data->rawQueueBuffers_.begin());
> + data->rawQueueBuffers_.push_back(std::move(temp));
data->video_->rawQueueBuffers_.pop();
Don't requeue the buffer; it'll come back to you in bufferReady.
> +
> + return 0;
> +}
> +
> +bool PipelineHandlerISP::match(DeviceEnumerator *enumerator)
> +{
> + DeviceMatch unicam("unicam");
> +
> + unicam.add("unicam-embedded");
> + unicam.add("unicam-image");
> +
> + MediaDevice *unicam_ = acquireMediaDevice(enumerator, unicam);
> + if (!unicam_) {
> + LOG(ISP, Debug) << "unicam Device not found";
> + return false;
> + }
> +
> + LOG(ISP, Debug) << "unicam Device Identified";
> +
> + std::unique_ptr<ISPCameraData> data = std::make_unique<ISPCameraData>(this, unicam_);
> +
> + if(data->init()) return false;
> +
> + std::set<Stream *> streams{&data->stream_};
> + std::shared_ptr<Camera> camera = Camera::create(this, data->video_->deviceName(), streams);
> + registerCamera(std::move(camera), std::move(data));
> +
> + return true;
> +}
> +
> +
> +void ISPCameraData::ISPCompleted(FrameBuffer *buffer)
> +{
> + Request *request = buffer->request();
You need to return the raw buffer to rawQueueBuffers_.
> +
> + pipe_->completeBuffer(request, buffer);
> + pipe_->completeRequest(request);
> +
> +}
> +
> +void ISPCameraData::bufferReady(FrameBuffer *buffer)
> +{
> + LOG(ISP, Debug) << rgbQueueBuffers_[0]->planes()[0].fd.fd();
> + isp_.invokeMethod(&CPU_ISP::processing, ConnectionTypeQueued, buffer, rgbQueueBuffers_[0], width_, height_);
FrameBuffer *rgbBuf = rgbQueueBuffers_.front();
isp_.invokeMethod(..., buffer, rgbBuf, ...);
> + rgbQueueBuffers_.erase(rgbQueueBuffers_.begin());
> + rgbQueueBuffers_.shrink_to_fit();
rgbQueueBuffers_.pop();
Almost there!
Paul
> +
> +}
> +
> +int ISPCameraData::init()
> +{
> + video_ = new V4L2VideoDevice(media_->getEntityByName("unicam-image"));
> + if (video_->open())
> + return -ENODEV;
> +
> + video_->bufferReady.connect(this, &ISPCameraData::bufferReady);
> + isp_.ispCompleted.connect(this, &ISPCameraData::ISPCompleted);
> +
> + return 0;
> +}
> +
> +REGISTER_PIPELINE_HANDLER(PipelineHandlerISP)
> +
> +} /* namespace libcamera */
> --
> 2.20.1
>
More information about the libcamera-devel
mailing list