[libcamera-devel] [PATCH v6 9/9] libcamera: pipeline: rkisp1: Attach to an IPA
Laurent Pinchart
laurent.pinchart at ideasonboard.com
Fri Oct 11 14:43:09 CEST 2019
Hi Niklas,
Thank you for the patch.
On Fri, Oct 11, 2019 at 05:22:16AM +0200, Niklas Söderlund wrote:
> Add the plumbing to the pipeline handler to interact with an IPA module.
> This change makes the usage of an IPA module mandatory for the rkisp1
> pipeline.
>
> The RkISP1 pipeline handler makes use of a timeline component to
> schedule actions. This might be useful for other pipeline handlers going
> forward so keep the generic timeline implementation separate to make it
> easy to break out.
I think there's indeed lots of room for improvement, but that can be
done on top of this series. For now this is a good enough proof of
concept.
Acked-by: Laurent Pinchart <laurent.pinchart at ideasonboard.com>
> Signed-off-by: Niklas Söderlund <niklas.soderlund at ragnatech.se>
> ---
> src/libcamera/pipeline/rkisp1/meson.build | 1 +
> src/libcamera/pipeline/rkisp1/rkisp1.cpp | 548 ++++++++++++++++++++-
> src/libcamera/pipeline/rkisp1/timeline.cpp | 227 +++++++++
> src/libcamera/pipeline/rkisp1/timeline.h | 72 +++
> 4 files changed, 831 insertions(+), 17 deletions(-)
> create mode 100644 src/libcamera/pipeline/rkisp1/timeline.cpp
> create mode 100644 src/libcamera/pipeline/rkisp1/timeline.h
>
> diff --git a/src/libcamera/pipeline/rkisp1/meson.build b/src/libcamera/pipeline/rkisp1/meson.build
> index f1cc4046b5d064cb..d04fb45223e72fa1 100644
> --- a/src/libcamera/pipeline/rkisp1/meson.build
> +++ b/src/libcamera/pipeline/rkisp1/meson.build
> @@ -1,3 +1,4 @@
> libcamera_sources += files([
> 'rkisp1.cpp',
> + 'timeline.cpp',
> ])
> diff --git a/src/libcamera/pipeline/rkisp1/rkisp1.cpp b/src/libcamera/pipeline/rkisp1/rkisp1.cpp
> index de4ab523d0e4fe36..029d5868d11f5bc9 100644
> --- a/src/libcamera/pipeline/rkisp1/rkisp1.cpp
> +++ b/src/libcamera/pipeline/rkisp1/rkisp1.cpp
> @@ -9,32 +9,115 @@
> #include <array>
> #include <iomanip>
> #include <memory>
> -#include <vector>
> +#include <queue>
>
> #include <linux/media-bus-format.h>
>
> +#include <ipa/rkisp1.h>
> +#include <libcamera/buffer.h>
> #include <libcamera/camera.h>
> +#include <libcamera/control_ids.h>
> #include <libcamera/request.h>
> #include <libcamera/stream.h>
>
> #include "camera_sensor.h"
> #include "device_enumerator.h"
> +#include "ipa_manager.h"
> #include "log.h"
> #include "media_device.h"
> #include "pipeline_handler.h"
> +#include "timeline.h"
> #include "utils.h"
> #include "v4l2_subdevice.h"
> #include "v4l2_videodevice.h"
>
> +#define RKISP1_PARAM_BASE 0x100
> +#define RKISP1_STAT_BASE 0x200
> +
> namespace libcamera {
>
> LOG_DEFINE_CATEGORY(RkISP1)
>
> +class PipelineHandlerRkISP1;
> +class RkISP1ActionQueueBuffers;
> +
> +enum RkISP1ActionType {
> + SetSensor,
> + SOE,
> + QueueBuffers,
> +};
> +
> +struct RkISP1FrameInfo {
> + unsigned int frame;
> + Request *request;
> +
> + Buffer *paramBuffer;
> + Buffer *statBuffer;
> + Buffer *videoBuffer;
> +
> + bool paramFilled;
> + bool paramDequeued;
> + bool metadataProcessed;
> +};
> +
> +class RkISP1Frames
> +{
> +public:
> + RkISP1Frames(PipelineHandler *pipe);
> +
> + RkISP1FrameInfo *create(unsigned int frame, Request *request, Stream *stream);
> + int destroy(unsigned int frame);
> +
> + RkISP1FrameInfo *find(unsigned int frame);
> + RkISP1FrameInfo *find(Buffer *buffer);
> + RkISP1FrameInfo *find(Request *request);
> +
> +private:
> + PipelineHandlerRkISP1 *pipe_;
> + std::map<unsigned int, RkISP1FrameInfo *> frameInfo_;
> +};
> +
> +class RkISP1Timeline : public Timeline
> +{
> +public:
> + RkISP1Timeline()
> + : Timeline()
> + {
> + setDelay(SetSensor, -1, 5);
> + setDelay(SOE, 0, -1);
> + setDelay(QueueBuffers, -1, 10);
> + }
> +
> + void bufferReady(Buffer *buffer)
> + {
> + /*
> + * Calculate SOE by taking the end of DMA set by the kernel and applying
> + * the time offsets provideprovided by the IPA to find the best estimate
> + * of SOE.
> + */
> +
> + ASSERT(frameOffset(SOE) == 0);
> +
> + utils::time_point soe = std::chrono::time_point<utils::clock>()
> + + std::chrono::nanoseconds(buffer->timestamp())
> + + timeOffset(SOE);
> +
> + notifyStartOfExposure(buffer->sequence(), soe);
> + }
> +
> + void setDelay(unsigned int type, int frame, int msdelay)
> + {
> + utils::duration delay = std::chrono::milliseconds(msdelay);
> + setRawDelay(type, frame, delay);
> + }
> +};
> +
> class RkISP1CameraData : public CameraData
> {
> public:
> RkISP1CameraData(PipelineHandler *pipe)
> - : CameraData(pipe), sensor_(nullptr)
> + : CameraData(pipe), sensor_(nullptr), frame_(0),
> + frameInfo_(pipe)
> {
> }
>
> @@ -43,8 +126,20 @@ public:
> delete sensor_;
> }
>
> + int loadIPA();
> +
> Stream stream_;
> CameraSensor *sensor_;
> + unsigned int frame_;
> + std::vector<IPABuffer> ipaBuffers_;
> + RkISP1Frames frameInfo_;
> + RkISP1Timeline timeline_;
> +
> +private:
> + void queueFrameAction(unsigned int frame,
> + const IPAOperationData &action);
> +
> + void metadataReady(unsigned int frame, const ControlList &metadata);
> };
>
> class RkISP1CameraConfiguration : public CameraConfiguration
> @@ -99,18 +194,235 @@ private:
> PipelineHandler::cameraData(camera));
> }
>
> + friend RkISP1ActionQueueBuffers;
> + friend RkISP1CameraData;
> + friend RkISP1Frames;
> +
> int initLinks();
> int createCamera(MediaEntity *sensor);
> + void tryCompleteRequest(Request *request);
> void bufferReady(Buffer *buffer);
> + void paramReady(Buffer *buffer);
> + void statReady(Buffer *buffer);
>
> MediaDevice *media_;
> V4L2Subdevice *dphy_;
> V4L2Subdevice *isp_;
> V4L2VideoDevice *video_;
> + V4L2VideoDevice *param_;
> + V4L2VideoDevice *stat_;
> +
> + BufferPool paramPool_;
> + BufferPool statPool_;
> +
> + std::queue<Buffer *> paramBuffers_;
> + std::queue<Buffer *> statBuffers_;
>
> Camera *activeCamera_;
> };
>
> +RkISP1Frames::RkISP1Frames(PipelineHandler *pipe)
> + : pipe_(dynamic_cast<PipelineHandlerRkISP1 *>(pipe))
> +{
> +}
> +
> +RkISP1FrameInfo *RkISP1Frames::create(unsigned int frame, Request *request, Stream *stream)
> +{
> + if (pipe_->paramBuffers_.empty()) {
> + LOG(RkISP1, Error) << "Parameters buffer underrun";
> + return nullptr;
> + }
> + Buffer *paramBuffer = pipe_->paramBuffers_.front();
> +
> + if (pipe_->statBuffers_.empty()) {
> + LOG(RkISP1, Error) << "Statisitc buffer underrun";
> + return nullptr;
> + }
> + Buffer *statBuffer = pipe_->statBuffers_.front();
> +
> + Buffer *videoBuffer = request->findBuffer(stream);
> + if (!videoBuffer) {
> + LOG(RkISP1, Error)
> + << "Attempt to queue request with invalid stream";
> + return nullptr;
> + }
> +
> + pipe_->paramBuffers_.pop();
> + pipe_->statBuffers_.pop();
> +
> + RkISP1FrameInfo *info = new RkISP1FrameInfo;
> +
> + info->frame = frame;
> + info->request = request;
> + info->paramBuffer = paramBuffer;
> + info->videoBuffer = videoBuffer;
> + info->statBuffer = statBuffer;
> + info->paramFilled = false;
> + info->paramDequeued = false;
> + info->metadataProcessed = false;
> +
> + frameInfo_[frame] = info;
> +
> + return info;
> +}
> +
> +int RkISP1Frames::destroy(unsigned int frame)
> +{
> + RkISP1FrameInfo *info = find(frame);
> + if (!info)
> + return -ENOENT;
> +
> + pipe_->paramBuffers_.push(info->paramBuffer);
> + pipe_->statBuffers_.push(info->statBuffer);
> +
> + frameInfo_.erase(info->frame);
> +
> + delete info;
> +
> + return 0;
> +}
> +
> +RkISP1FrameInfo *RkISP1Frames::find(unsigned int frame)
> +{
> + auto itInfo = frameInfo_.find(frame);
> +
> + if (itInfo != frameInfo_.end())
> + return itInfo->second;
> +
> + LOG(RkISP1, Error) << "Can't locate info from frame";
> + return nullptr;
> +}
> +
> +RkISP1FrameInfo *RkISP1Frames::find(Buffer *buffer)
> +{
> + for (auto &itInfo : frameInfo_) {
> + RkISP1FrameInfo *info = itInfo.second;
> +
> + if (info->paramBuffer == buffer ||
> + info->statBuffer == buffer ||
> + info->videoBuffer == buffer)
> + return info;
> + }
> +
> + LOG(RkISP1, Error) << "Can't locate info from buffer";
> + return nullptr;
> +}
> +
> +RkISP1FrameInfo *RkISP1Frames::find(Request *request)
> +{
> + for (auto &itInfo : frameInfo_) {
> + RkISP1FrameInfo *info = itInfo.second;
> +
> + if (info->request == request)
> + return info;
> + }
> +
> + LOG(RkISP1, Error) << "Can't locate info from request";
> + return nullptr;
> +}
> +
> +class RkISP1ActionSetSensor : public FrameAction
> +{
> +public:
> + RkISP1ActionSetSensor(unsigned int frame, CameraSensor *sensor, V4L2ControlList controls)
> + : FrameAction(frame, SetSensor), sensor_(sensor), controls_(controls) {}
> +
> +protected:
> + void run() override
> + {
> + sensor_->setControls(&controls_);
> + }
> +
> +private:
> + CameraSensor *sensor_;
> + V4L2ControlList controls_;
> +};
> +
> +class RkISP1ActionQueueBuffers : public FrameAction
> +{
> +public:
> + RkISP1ActionQueueBuffers(unsigned int frame, RkISP1CameraData *data,
> + PipelineHandlerRkISP1 *pipe)
> + : FrameAction(frame, QueueBuffers), data_(data), pipe_(pipe)
> + {
> + }
> +
> +protected:
> + void run() override
> + {
> + RkISP1FrameInfo *info = data_->frameInfo_.find(frame());
> + if (!info)
> + LOG(RkISP1, Fatal) << "Frame not known";
> +
> + if (info->paramFilled)
> + pipe_->param_->queueBuffer(info->paramBuffer);
> + else
> + LOG(RkISP1, Error)
> + << "Parameters not ready on time for frame "
> + << frame() << ", ignore parameters.";
> +
> + pipe_->stat_->queueBuffer(info->statBuffer);
> + pipe_->video_->queueBuffer(info->videoBuffer);
> + }
> +
> +private:
> + RkISP1CameraData *data_;
> + PipelineHandlerRkISP1 *pipe_;
> +};
> +
> +int RkISP1CameraData::loadIPA()
> +{
> + ipa_ = IPAManager::instance()->createIPA(pipe_, 1, 1);
> + if (!ipa_)
> + return -ENOENT;
> +
> + ipa_->queueFrameAction.connect(this,
> + &RkISP1CameraData::queueFrameAction);
> +
> + return 0;
> +}
> +
> +void RkISP1CameraData::queueFrameAction(unsigned int frame,
> + const IPAOperationData &action)
> +{
> + switch (action.operation) {
> + case RKISP1_IPA_ACTION_V4L2_SET: {
> + V4L2ControlList controls = action.v4l2controls[0];
> + timeline_.scheduleAction(utils::make_unique<RkISP1ActionSetSensor>(frame,
> + sensor_,
> + controls));
> + break;
> + }
> + case RKISP1_IPA_ACTION_PARAM_FILLED: {
> + RkISP1FrameInfo *info = frameInfo_.find(frame);
> + if (info)
> + info->paramFilled = true;
> + break;
> + }
> + case RKISP1_IPA_ACTION_METADATA:
> + metadataReady(frame, action.controls[0]);
> + break;
> + default:
> + LOG(RkISP1, Error) << "Unkown action " << action.operation;
> + break;
> + }
> +}
> +
> +void RkISP1CameraData::metadataReady(unsigned int frame, const ControlList &metadata)
> +{
> + PipelineHandlerRkISP1 *pipe =
> + static_cast<PipelineHandlerRkISP1 *>(pipe_);
> +
> + RkISP1FrameInfo *info = frameInfo_.find(frame);
> + if (!info)
> + return;
> +
> + info->request->metadata() = metadata;
> + info->metadataProcessed = true;
> +
> + pipe->tryCompleteRequest(info->request);
> +}
> +
> RkISP1CameraConfiguration::RkISP1CameraConfiguration(Camera *camera,
> RkISP1CameraData *data)
> : CameraConfiguration()
> @@ -202,12 +514,14 @@ CameraConfiguration::Status RkISP1CameraConfiguration::validate()
>
> PipelineHandlerRkISP1::PipelineHandlerRkISP1(CameraManager *manager)
> : PipelineHandler(manager), dphy_(nullptr), isp_(nullptr),
> - video_(nullptr)
> + video_(nullptr), param_(nullptr), stat_(nullptr)
> {
> }
>
> PipelineHandlerRkISP1::~PipelineHandlerRkISP1()
> {
> + delete param_;
> + delete stat_;
> delete video_;
> delete isp_;
> delete dphy_;
> @@ -324,6 +638,18 @@ int PipelineHandlerRkISP1::configure(Camera *camera, CameraConfiguration *c)
> return -EINVAL;
> }
>
> + V4L2DeviceFormat paramFormat = {};
> + paramFormat.fourcc = V4L2_META_FMT_RK_ISP1_PARAMS;
> + ret = param_->setFormat(¶mFormat);
> + if (ret)
> + return ret;
> +
> + V4L2DeviceFormat statFormat = {};
> + statFormat.fourcc = V4L2_META_FMT_RK_ISP1_STAT_3A;
> + ret = stat_->setFormat(&statFormat);
> + if (ret)
> + return ret;
> +
> cfg.setStream(&data->stream_);
>
> return 0;
> @@ -332,39 +658,135 @@ int PipelineHandlerRkISP1::configure(Camera *camera, CameraConfiguration *c)
> int PipelineHandlerRkISP1::allocateBuffers(Camera *camera,
> const std::set<Stream *> &streams)
> {
> + RkISP1CameraData *data = cameraData(camera);
> Stream *stream = *streams.begin();
> + int ret;
>
> if (stream->memoryType() == InternalMemory)
> - return video_->exportBuffers(&stream->bufferPool());
> + ret = video_->exportBuffers(&stream->bufferPool());
> else
> - return video_->importBuffers(&stream->bufferPool());
> + ret = video_->importBuffers(&stream->bufferPool());
> +
> + if (ret)
> + return ret;
> +
> + paramPool_.createBuffers(stream->configuration().bufferCount + 1);
> + ret = param_->exportBuffers(¶mPool_);
> + if (ret) {
> + video_->releaseBuffers();
> + return ret;
> + }
> +
> + statPool_.createBuffers(stream->configuration().bufferCount + 1);
> + ret = stat_->exportBuffers(&statPool_);
> + if (ret) {
> + param_->releaseBuffers();
> + video_->releaseBuffers();
> + return ret;
> + }
> +
> + for (unsigned int i = 0; i < stream->configuration().bufferCount + 1; i++) {
> + data->ipaBuffers_.push_back({ .id = RKISP1_PARAM_BASE | i,
> + .memory = paramPool_.buffers()[i] });
> + paramBuffers_.push(new Buffer(i));
> + }
> +
> + for (unsigned int i = 0; i < stream->configuration().bufferCount + 1; i++) {
> + data->ipaBuffers_.push_back({ .id = RKISP1_STAT_BASE | i,
> + .memory = statPool_.buffers()[i] });
> + statBuffers_.push(new Buffer(i));
> + }
> +
> + data->ipa_->mapBuffers(data->ipaBuffers_);
> +
> + return ret;
> }
>
> int PipelineHandlerRkISP1::freeBuffers(Camera *camera,
> const std::set<Stream *> &streams)
> {
> + RkISP1CameraData *data = cameraData(camera);
> +
> + while (!statBuffers_.empty()) {
> + delete statBuffers_.front();
> + statBuffers_.pop();
> + }
> +
> + while (!paramBuffers_.empty()) {
> + delete paramBuffers_.front();
> + paramBuffers_.pop();
> + }
> +
> + std::vector<unsigned int> ids;
> + for (IPABuffer &ipabuf : data->ipaBuffers_)
> + ids.push_back(ipabuf.id);
> +
> + data->ipa_->unmapBuffers(ids);
> + data->ipaBuffers_.clear();
> +
> + if (param_->releaseBuffers())
> + LOG(RkISP1, Error) << "Failed to release parameters buffers";
> +
> + if (stat_->releaseBuffers())
> + LOG(RkISP1, Error) << "Failed to release stat buffers";
> +
> if (video_->releaseBuffers())
> - LOG(RkISP1, Error) << "Failed to release buffers";
> + LOG(RkISP1, Error) << "Failed to release video buffers";
>
> return 0;
> }
>
> int PipelineHandlerRkISP1::start(Camera *camera)
> {
> + RkISP1CameraData *data = cameraData(camera);
> int ret;
>
> + data->frame_ = 0;
> +
> + ret = param_->streamOn();
> + if (ret) {
> + LOG(RkISP1, Error)
> + << "Failed to start parameters " << camera->name();
> + return ret;
> + }
> +
> + ret = stat_->streamOn();
> + if (ret) {
> + param_->streamOff();
> + LOG(RkISP1, Error)
> + << "Failed to start statistics " << camera->name();
> + return ret;
> + }
> +
> ret = video_->streamOn();
> - if (ret)
> + if (ret) {
> + param_->streamOff();
> + stat_->streamOff();
> +
> LOG(RkISP1, Error)
> << "Failed to start camera " << camera->name();
> + }
>
> activeCamera_ = camera;
>
> + /* Inform IPA of stream configuration and sensor controls. */
> + std::map<unsigned int, IPAStream> streamConfig;
> + streamConfig[0] = {
> + .pixelFormat = data->stream_.configuration().pixelFormat,
> + .size = data->stream_.configuration().size,
> + };
> +
> + std::map<unsigned int, V4L2ControlInfoMap> entityControls;
> + entityControls[0] = data->sensor_->controls();
> +
> + data->ipa_->configure(streamConfig, entityControls);
> +
> return ret;
> }
>
> void PipelineHandlerRkISP1::stop(Camera *camera)
> {
> + RkISP1CameraData *data = cameraData(camera);
> int ret;
>
> ret = video_->streamOff();
> @@ -372,6 +794,18 @@ void PipelineHandlerRkISP1::stop(Camera *camera)
> LOG(RkISP1, Warning)
> << "Failed to stop camera " << camera->name();
>
> + ret = stat_->streamOff();
> + if (ret)
> + LOG(RkISP1, Warning)
> + << "Failed to stop statistics " << camera->name();
> +
> + ret = param_->streamOff();
> + if (ret)
> + LOG(RkISP1, Warning)
> + << "Failed to stop parameters " << camera->name();
> +
> + data->timeline_.reset();
> +
> activeCamera_ = nullptr;
> }
>
> @@ -380,18 +814,24 @@ int PipelineHandlerRkISP1::queueRequest(Camera *camera, Request *request)
> RkISP1CameraData *data = cameraData(camera);
> Stream *stream = &data->stream_;
>
> - Buffer *buffer = request->findBuffer(stream);
> - if (!buffer) {
> - LOG(RkISP1, Error)
> - << "Attempt to queue request with invalid stream";
> + PipelineHandler::queueRequest(camera, request);
> +
> + RkISP1FrameInfo *info = data->frameInfo_.create(data->frame_, request,
> + stream);
> + if (!info)
> return -ENOENT;
> - }
>
> - int ret = video_->queueBuffer(buffer);
> - if (ret < 0)
> - return ret;
> + IPAOperationData op;
> + op.operation = RKISP1_IPA_EVENT_QUEUE_REQUEST;
> + op.data = { data->frame_, RKISP1_PARAM_BASE | info->paramBuffer->index() };
> + op.controls = { request->controls() };
> + data->ipa_->processEvent(op);
>
> - PipelineHandler::queueRequest(camera, request);
> + data->timeline_.scheduleAction(utils::make_unique<RkISP1ActionQueueBuffers>(data->frame_,
> + data,
> + this));
> +
> + data->frame_++;
>
> return 0;
> }
> @@ -435,11 +875,19 @@ int PipelineHandlerRkISP1::createCamera(MediaEntity *sensor)
> std::unique_ptr<RkISP1CameraData> data =
> utils::make_unique<RkISP1CameraData>(this);
>
> + data->controlInfo_.emplace(std::piecewise_construct,
> + std::forward_as_tuple(&controls::AeEnable),
> + std::forward_as_tuple(false, true));
> +
> data->sensor_ = new CameraSensor(sensor);
> ret = data->sensor_->init();
> if (ret)
> return ret;
>
> + ret = data->loadIPA();
> + if (ret)
> + return ret;
> +
> std::set<Stream *> streams{ &data->stream_ };
> std::shared_ptr<Camera> camera =
> Camera::create(this, sensor->name(), streams);
> @@ -478,7 +926,17 @@ bool PipelineHandlerRkISP1::match(DeviceEnumerator *enumerator)
> if (video_->open() < 0)
> return false;
>
> + stat_ = V4L2VideoDevice::fromEntityName(media_, "rkisp1-statistics");
> + if (stat_->open() < 0)
> + return false;
> +
> + param_ = V4L2VideoDevice::fromEntityName(media_, "rkisp1-input-params");
> + if (param_->open() < 0)
> + return false;
> +
> video_->bufferReady.connect(this, &PipelineHandlerRkISP1::bufferReady);
> + stat_->bufferReady.connect(this, &PipelineHandlerRkISP1::statReady);
> + param_->bufferReady.connect(this, &PipelineHandlerRkISP1::paramReady);
>
> /* Configure default links. */
> if (initLinks() < 0) {
> @@ -504,13 +962,69 @@ bool PipelineHandlerRkISP1::match(DeviceEnumerator *enumerator)
> * Buffer Handling
> */
>
> +void PipelineHandlerRkISP1::tryCompleteRequest(Request *request)
> +{
> + RkISP1CameraData *data = cameraData(activeCamera_);
> + RkISP1FrameInfo *info = data->frameInfo_.find(request);
> + if (!info)
> + return;
> +
> + if (request->hasPendingBuffers())
> + return;
> +
> + if (!info->metadataProcessed)
> + return;
> +
> + if (!info->paramDequeued)
> + return;
> +
> + completeRequest(activeCamera_, request);
> +
> + data->frameInfo_.destroy(info->frame);
> +}
> +
> void PipelineHandlerRkISP1::bufferReady(Buffer *buffer)
> {
> ASSERT(activeCamera_);
> + RkISP1CameraData *data = cameraData(activeCamera_);
> Request *request = buffer->request();
>
> + data->timeline_.bufferReady(buffer);
> +
> + if (data->frame_ <= buffer->sequence())
> + data->frame_ = buffer->sequence() + 1;
> +
> completeBuffer(activeCamera_, request, buffer);
> - completeRequest(activeCamera_, request);
> + tryCompleteRequest(request);
> +}
> +
> +void PipelineHandlerRkISP1::paramReady(Buffer *buffer)
> +{
> + ASSERT(activeCamera_);
> + RkISP1CameraData *data = cameraData(activeCamera_);
> +
> + RkISP1FrameInfo *info = data->frameInfo_.find(buffer);
> +
> + info->paramDequeued = true;
> + tryCompleteRequest(info->request);
> +}
> +
> +void PipelineHandlerRkISP1::statReady(Buffer *buffer)
> +{
> + ASSERT(activeCamera_);
> + RkISP1CameraData *data = cameraData(activeCamera_);
> +
> + RkISP1FrameInfo *info = data->frameInfo_.find(buffer);
> + if (!info)
> + return;
> +
> + unsigned int frame = info->frame;
> + unsigned int statid = RKISP1_STAT_BASE | info->statBuffer->index();
> +
> + IPAOperationData op;
> + op.operation = RKISP1_IPA_EVENT_SIGNAL_STAT_BUFFER;
> + op.data = { frame, statid };
> + data->ipa_->processEvent(op);
> }
>
> REGISTER_PIPELINE_HANDLER(PipelineHandlerRkISP1);
> diff --git a/src/libcamera/pipeline/rkisp1/timeline.cpp b/src/libcamera/pipeline/rkisp1/timeline.cpp
> new file mode 100644
> index 0000000000000000..b98a16689fa994fe
> --- /dev/null
> +++ b/src/libcamera/pipeline/rkisp1/timeline.cpp
> @@ -0,0 +1,227 @@
> +/* SPDX-License-Identifier: LGPL-2.1-or-later */
> +/*
> + * Copyright (C) 2019, Google Inc.
> + *
> + * timeline.cpp - Timeline for per-frame control
> + */
> +
> +#include "timeline.h"
> +
> +#include "log.h"
> +
> +/**
> + * \file timeline.h
> + * \brief Timeline for per-frame control
> + */
> +
> +namespace libcamera {
> +
> +LOG_DEFINE_CATEGORY(Timeline)
> +
> +/**
> + * \class FrameAction
> + * \brief Action that can be schedule on a Timeline
> + *
> + * A frame action is an event schedule to be executed on a Timeline. A frame
> + * action has two primal attributes a frame number and a type.
> + *
> + * The frame number describes the frame to which the action is associated. The
> + * type is a numerical ID which identifies the action within the pipeline and
> + * IPA protocol.
> + */
> +
> +/**
> + * \class Timeline
> + * \brief Executor of FrameAction
> + *
> + * The timeline has three primary functions:
> + *
> + * 1. Keep track of the Start of Exposure (SOE) for every frame processed by
> + * the hardware. Using this information it shall keep an up-to-date estimate
> + * of the frame interval (time between two consecutive SOE events).
> + *
> + * The estimated frame interval together with recorded SOE events are the
> + * foundation for how the timeline schedule FrameAction at specific points
> + * in time.
> + * \todo Improve the frame interval estimation algorithm.
> + *
> + * 2. Keep track of current delays for different types of actions. The delays
> + * for different actions might differ during a capture session. Exposure time
> + * effects the over all FPS and different ISP parameters might impacts its
> + * processing time.
> + *
> + * The action type delays shall be updated by the IPA in conjunction with
> + * how it changes the capture parameters.
> + *
> + * 3. Schedule actions on the timeline. This is the process of taking a
> + * FrameAction which contains an abstract description of what frame and
> + * what type of action it contains and turning that into an time point
> + * and make sure the action is executed at that time.
> + */
> +
> +Timeline::Timeline()
> + : frameInterval_(0)
> +{
> + timer_.timeout.connect(this, &Timeline::timeout);
> +}
> +
> +/**
> + * \brief Reset and stop the timeline
> + *
> + * The timeline needs to be reset when the timeline should no longer execute
> + * actions. A timeline should be reset between two capture sessions to prevent
> + * the old capture session to effect the second one.
> + */
> +void Timeline::reset()
> +{
> + timer_.stop();
> +
> + actions_.clear();
> + history_.clear();
> +}
> +
> +/**
> + * \brief Schedule an action on the timeline
> + * \param[in] action FrameAction to schedule
> + *
> + * The act of scheduling an action to the timeline is the process of taking
> + * the properties of the action (type, frame and time offsets) and translating
> + * that to a time point using the current values for the action type timings
> + * value recorded in the timeline. If an action is scheduled too late, execute
> + * it immediately.
> + */
> +void Timeline::scheduleAction(std::unique_ptr<FrameAction> action)
> +{
> + unsigned int lastFrame;
> + utils::time_point lastTime;
> +
> + if (history_.empty()) {
> + lastFrame = 0;
> + lastTime = std::chrono::steady_clock::now();
> + } else {
> + lastFrame = history_.back().first;
> + lastTime = history_.back().second;
> + }
> +
> + /*
> + * Calculate when the action shall be schedule by first finding out how
> + * many frames in the future the action acts on and then add the actions
> + * frame offset. After the spatial frame offset is found out translate
> + * that to a time point by using the last estimated start of exposure
> + * (SOE) as the fixed offset. Lastly add the action time offset to the
> + * time point.
> + */
> + int frame = action->frame() - lastFrame + frameOffset(action->type());
> + utils::time_point deadline = lastTime + frame * frameInterval_
> + + timeOffset(action->type());
> +
> + utils::time_point now = std::chrono::steady_clock::now();
> + if (deadline < now) {
> + LOG(Timeline, Warning)
> + << "Action scheduled too late "
> + << utils::time_point_to_string(deadline)
> + << ", run now " << utils::time_point_to_string(now);
> + action->run();
> + } else {
> + actions_.insert({ deadline, std::move(action) });
> + updateDeadline();
> + }
> +}
> +
> +void Timeline::notifyStartOfExposure(unsigned int frame, utils::time_point time)
> +{
> + history_.push_back(std::make_pair(frame, time));
> +
> + if (history_.size() <= HISTORY_DEPTH / 2)
> + return;
> +
> + while (history_.size() > HISTORY_DEPTH)
> + history_.pop_front();
> +
> + /* Update esitmated time between two start of exposures. */
> + utils::duration sumExposures(0);
> + unsigned int numExposures = 0;
> +
> + utils::time_point lastTime;
> + for (auto it = history_.begin(); it != history_.end(); it++) {
> + if (it != history_.begin()) {
> + sumExposures += it->second - lastTime;
> + numExposures++;
> + }
> +
> + lastTime = it->second;
> + }
> +
> + frameInterval_ = sumExposures;
> + if (numExposures)
> + frameInterval_ /= numExposures;
> +}
> +
> +int Timeline::frameOffset(unsigned int type) const
> +{
> + const auto it = delays_.find(type);
> + if (it == delays_.end()) {
> + LOG(Timeline, Error)
> + << "No frame offset set for action type " << type;
> + return 0;
> + }
> +
> + return it->second.first;
> +}
> +
> +utils::duration Timeline::timeOffset(unsigned int type) const
> +{
> + const auto it = delays_.find(type);
> + if (it == delays_.end()) {
> + LOG(Timeline, Error)
> + << "No time offset set for action type " << type;
> + return utils::duration::zero();
> + }
> +
> + return it->second.second;
> +}
> +
> +void Timeline::setRawDelay(unsigned int type, int frame, utils::duration time)
> +{
> + delays_[type] = std::make_pair(frame, time);
> +}
> +
> +void Timeline::updateDeadline()
> +{
> + if (actions_.empty())
> + return;
> +
> + const utils::time_point &deadline = actions_.begin()->first;
> +
> + if (timer_.isRunning() && deadline >= timer_.deadline())
> + return;
> +
> + if (deadline <= std::chrono::steady_clock::now()) {
> + timeout(&timer_);
> + return;
> + }
> +
> + timer_.start(deadline);
> +}
> +
> +void Timeline::timeout(Timer *timer)
> +{
> + utils::time_point now = std::chrono::steady_clock::now();
> +
> + for (auto it = actions_.begin(); it != actions_.end();) {
> + const utils::time_point &sched = it->first;
> +
> + if (sched > now)
> + break;
> +
> + FrameAction *action = it->second.get();
> +
> + action->run();
> +
> + it = actions_.erase(it);
> + }
> +
> + updateDeadline();
> +}
> +
> +} /* namespace libcamera */
> diff --git a/src/libcamera/pipeline/rkisp1/timeline.h b/src/libcamera/pipeline/rkisp1/timeline.h
> new file mode 100644
> index 0000000000000000..9d30e4eaf8743d07
> --- /dev/null
> +++ b/src/libcamera/pipeline/rkisp1/timeline.h
> @@ -0,0 +1,72 @@
> +/* SPDX-License-Identifier: LGPL-2.1-or-later */
> +/*
> + * Copyright (C) 2019, Google Inc.
> + *
> + * timeline.h - Timeline for per-frame controls
> + */
> +#ifndef __LIBCAMERA_TIMELINE_H__
> +#define __LIBCAMERA_TIMELINE_H__
> +
> +#include <list>
> +#include <map>
> +
> +#include <libcamera/timer.h>
> +
> +#include "utils.h"
> +
> +namespace libcamera {
> +
> +class FrameAction
> +{
> +public:
> + FrameAction(unsigned int frame, unsigned int type)
> + : frame_(frame), type_(type) {}
> +
> + virtual ~FrameAction() {}
> +
> + unsigned int frame() const { return frame_; }
> + unsigned int type() const { return type_; }
> +
> + virtual void run() = 0;
> +
> +private:
> + unsigned int frame_;
> + unsigned int type_;
> +};
> +
> +class Timeline
> +{
> +public:
> + Timeline();
> + virtual ~Timeline() {}
> +
> + virtual void reset();
> + virtual void scheduleAction(std::unique_ptr<FrameAction> action);
> + virtual void notifyStartOfExposure(unsigned int frame, utils::time_point time);
> +
> + utils::duration frameInterval() const { return frameInterval_; }
> +
> +protected:
> + int frameOffset(unsigned int type) const;
> + utils::duration timeOffset(unsigned int type) const;
> +
> + void setRawDelay(unsigned int type, int frame, utils::duration time);
> +
> + std::map<unsigned int, std::pair<int, utils::duration>> delays_;
> +
> +private:
> + static constexpr unsigned int HISTORY_DEPTH = 10;
> +
> + void timeout(Timer *timer);
> + void updateDeadline();
> +
> + std::list<std::pair<unsigned int, utils::time_point>> history_;
> + std::multimap<utils::time_point, std::unique_ptr<FrameAction>> actions_;
> + utils::duration frameInterval_;
> +
> + Timer timer_;
> +};
> +
> +} /* namespace libcamera */
> +
> +#endif /* __LIBCAMERA_TIMELINE_H__ */
--
Regards,
Laurent Pinchart
More information about the libcamera-devel
mailing list