[libcamera-devel] [PATCH v4 11/11] libcamera: pipeline: rkisp1: Attach to an IPA
Niklas Söderlund
niklas.soderlund at ragnatech.se
Thu Oct 3 19:49:41 CEST 2019
Add the plumbing to the pipeline handler to interact with an IPA module.
This change makes the usage of an IPA module mandatory for the rkisp1
pipeline.
Signed-off-by: Niklas Söderlund <niklas.soderlund at ragnatech.se>
---
src/libcamera/pipeline/rkisp1/meson.build | 1 +
src/libcamera/pipeline/rkisp1/rkisp1.cpp | 326 ++++++++++++++++++++-
src/libcamera/pipeline/rkisp1/rkisp1.h | 78 +++++
src/libcamera/pipeline/rkisp1/timeline.cpp | 56 ++++
4 files changed, 448 insertions(+), 13 deletions(-)
create mode 100644 src/libcamera/pipeline/rkisp1/rkisp1.h
create mode 100644 src/libcamera/pipeline/rkisp1/timeline.cpp
diff --git a/src/libcamera/pipeline/rkisp1/meson.build b/src/libcamera/pipeline/rkisp1/meson.build
index f1cc4046b5d064cb..d04fb45223e72fa1 100644
--- a/src/libcamera/pipeline/rkisp1/meson.build
+++ b/src/libcamera/pipeline/rkisp1/meson.build
@@ -1,3 +1,4 @@
libcamera_sources += files([
'rkisp1.cpp',
+ 'timeline.cpp',
])
diff --git a/src/libcamera/pipeline/rkisp1/rkisp1.cpp b/src/libcamera/pipeline/rkisp1/rkisp1.cpp
index de4ab523d0e4fe36..ade8c8823a0ad99d 100644
--- a/src/libcamera/pipeline/rkisp1/rkisp1.cpp
+++ b/src/libcamera/pipeline/rkisp1/rkisp1.cpp
@@ -5,20 +5,22 @@
* rkisp1.cpp - Pipeline handler for Rockchip ISP1
*/
+#include "rkisp1.h"
+
#include <algorithm>
#include <array>
#include <iomanip>
#include <memory>
-#include <vector>
#include <linux/media-bus-format.h>
#include <libcamera/camera.h>
+#include <libcamera/control_ids.h>
#include <libcamera/request.h>
#include <libcamera/stream.h>
-#include "camera_sensor.h"
#include "device_enumerator.h"
+#include "ipa_manager.h"
#include "log.h"
#include "media_device.h"
#include "pipeline_handler.h"
@@ -34,7 +36,7 @@ class RkISP1CameraData : public CameraData
{
public:
RkISP1CameraData(PipelineHandler *pipe)
- : CameraData(pipe), sensor_(nullptr)
+ : CameraData(pipe), sensor_(nullptr), frame_(0)
{
}
@@ -43,8 +45,21 @@ public:
delete sensor_;
}
+ int loadIPA();
+
Stream stream_;
CameraSensor *sensor_;
+ unsigned int frame_;
+ std::vector<IPABuffer> ipaBuffers_;
+ std::map<unsigned int, Request *> frameInfo_;
+ RkISP1Timeline timeline_;
+
+private:
+ void queueFrameAction(const IPAOperationData &action);
+
+ void queueBuffer(unsigned int frame, unsigned int type,
+ unsigned int id);
+ void metaDataReady(unsigned int frame, unsigned int aeState);
};
class RkISP1CameraConfiguration : public CameraConfiguration
@@ -99,18 +114,114 @@ private:
PipelineHandler::cameraData(camera));
}
+ friend RkISP1CameraData;
+
int initLinks();
int createCamera(MediaEntity *sensor);
+ void tryCompleteRequest(Request *request);
void bufferReady(Buffer *buffer);
+ void paramReady(Buffer *buffer);
+ void statReady(Buffer *buffer);
MediaDevice *media_;
V4L2Subdevice *dphy_;
V4L2Subdevice *isp_;
V4L2VideoDevice *video_;
+ V4L2VideoDevice *param_;
+ V4L2VideoDevice *stat_;
+
+ BufferPool paramPool_;
+ BufferPool statPool_;
+
+ std::map<unsigned int, Buffer *> paramBuffers_;
+ std::map<unsigned int, Buffer *> statBuffers_;
Camera *activeCamera_;
};
+int RkISP1CameraData::loadIPA()
+{
+ ipa_ = IPAManager::instance()->createIPA(pipe_, 1, 1);
+ if (!ipa_)
+ return -ENOENT;
+
+ ipa_->queueFrameAction.connect(this,
+ &RkISP1CameraData::queueFrameAction);
+
+ return 0;
+}
+
+void RkISP1CameraData::queueFrameAction(const IPAOperationData &action)
+{
+ switch (action.operation) {
+ case RKISP1_IPA_ACTION_V4L2_SET: {
+ unsigned int frame = action.data[0];
+ V4L2ControlList controls;
+ for (unsigned int i = 1; i < action.data.size(); i += 2)
+ controls.add(action.data[i], action.data[i + 1]);
+ timeline_.scheduleAction(new RkISP1ActionSetSensor(frame, sensor_, controls));
+ break;
+ }
+ case RKISP1_IPA_ACTION_QUEUE_BUFFER: {
+ unsigned int frame = action.data[0];
+ unsigned int type = action.data[1];
+ unsigned int id = action.data[2];
+ queueBuffer(frame, type, id);
+ break;
+ }
+ case RKISP1_IPA_ACTION_META_DATA: {
+ unsigned int frame = action.data[0];
+ unsigned aeState = action.data[1];
+ metaDataReady(frame, aeState);
+ break;
+ }
+ default:
+ LOG(RkISP1, Error) << "Unkown action " << action.operation;
+ break;
+ }
+}
+
+void RkISP1CameraData::queueBuffer(unsigned int frame, unsigned int type,
+ unsigned int id)
+{
+ PipelineHandlerRkISP1 *pipe =
+ static_cast<PipelineHandlerRkISP1 *>(pipe_);
+
+ RkISP1ActionType acttype;
+ V4L2VideoDevice *device;
+ Buffer *buffer;
+ switch (type) {
+ case RKISP1_BUFFER_PARAM:
+ acttype = QueueParameters;
+ device = pipe->param_;
+ buffer = pipe->paramBuffers_[id];
+ break;
+ case RKISP1_BUFFER_STAT:
+ acttype = QueueStatistics;
+ device = pipe->stat_;
+ buffer = pipe->statBuffers_[id];
+ break;
+ default:
+ LOG(RkISP1, Error) << "Unkown IPA buffer type " << type;
+ return;
+ }
+
+ timeline_.scheduleAction(new RkISP1ActionQueueBuffer(frame, acttype,
+ device, buffer));
+}
+
+void RkISP1CameraData::metaDataReady(unsigned int frame, unsigned int aeStatus)
+{
+ Request *request = frameInfo_[frame];
+ PipelineHandlerRkISP1 *pipe =
+ static_cast<PipelineHandlerRkISP1 *>(pipe_);
+
+ if (aeStatus)
+ request->metaData().set<bool>(controls::AeLocked, aeStatus == 2);
+
+ pipe->tryCompleteRequest(request);
+}
+
RkISP1CameraConfiguration::RkISP1CameraConfiguration(Camera *camera,
RkISP1CameraData *data)
: CameraConfiguration()
@@ -202,12 +313,14 @@ CameraConfiguration::Status RkISP1CameraConfiguration::validate()
PipelineHandlerRkISP1::PipelineHandlerRkISP1(CameraManager *manager)
: PipelineHandler(manager), dphy_(nullptr), isp_(nullptr),
- video_(nullptr)
+ video_(nullptr), param_(nullptr), stat_(nullptr)
{
}
PipelineHandlerRkISP1::~PipelineHandlerRkISP1()
{
+ delete param_;
+ delete stat_;
delete video_;
delete isp_;
delete dphy_;
@@ -317,6 +430,20 @@ int PipelineHandlerRkISP1::configure(Camera *camera, CameraConfiguration *c)
if (ret)
return ret;
+ V4L2DeviceFormat paramFormat = {};
+ paramFormat.fourcc = V4L2_META_FMT_RK_ISP1_PARAMS;
+
+ ret = param_->setFormat(¶mFormat);
+ if (ret)
+ return ret;
+
+ V4L2DeviceFormat statFormat = {};
+ statFormat.fourcc = V4L2_META_FMT_RK_ISP1_STAT_3A;
+
+ ret = stat_->setFormat(&statFormat);
+ if (ret)
+ return ret;
+
if (outputFormat.size != cfg.size ||
outputFormat.fourcc != cfg.pixelFormat) {
LOG(RkISP1, Error)
@@ -332,39 +459,133 @@ int PipelineHandlerRkISP1::configure(Camera *camera, CameraConfiguration *c)
int PipelineHandlerRkISP1::allocateBuffers(Camera *camera,
const std::set<Stream *> &streams)
{
+ RkISP1CameraData *data = cameraData(camera);
Stream *stream = *streams.begin();
+ int ret;
if (stream->memoryType() == InternalMemory)
- return video_->exportBuffers(&stream->bufferPool());
+ ret = video_->exportBuffers(&stream->bufferPool());
else
- return video_->importBuffers(&stream->bufferPool());
+ ret = video_->importBuffers(&stream->bufferPool());
+
+ if (ret)
+ return ret;
+
+ paramPool_.createBuffers(stream->configuration().bufferCount + 1);
+ ret = param_->exportBuffers(¶mPool_);
+ if (ret) {
+ video_->releaseBuffers();
+ return ret;
+ }
+
+ statPool_.createBuffers(stream->configuration().bufferCount + 1);
+ ret = stat_->exportBuffers(&statPool_);
+ if (ret) {
+ param_->releaseBuffers();
+ video_->releaseBuffers();
+ return ret;
+ }
+
+ for (unsigned int i = 0; i < stream->configuration().bufferCount + 1; i++) {
+ paramBuffers_[i] = new Buffer(i);
+ statBuffers_[i] = new Buffer(i);
+
+ data->ipaBuffers_.push_back({
+ .id = i,
+ .type = RKISP1_BUFFER_PARAM,
+ .buffer = paramPool_.buffers()[i],
+ });
+ data->ipaBuffers_.push_back({
+ .id = i,
+ .type = RKISP1_BUFFER_STAT,
+ .buffer = statPool_.buffers()[i],
+ });
+ }
+
+ data->ipa_->mapBuffers(data->ipaBuffers_);
+
+ return ret;
}
int PipelineHandlerRkISP1::freeBuffers(Camera *camera,
const std::set<Stream *> &streams)
{
+ RkISP1CameraData *data = cameraData(camera);
+
+ data->ipa_->unmapBuffers(data->ipaBuffers_);
+ data->ipaBuffers_.clear();
+
+ for (auto it : paramBuffers_)
+ delete it.second;
+
+ paramBuffers_.clear();
+
+ for (auto it : statBuffers_)
+ delete it.second;
+
+ statBuffers_.clear();
+
+ if (param_->releaseBuffers())
+ LOG(RkISP1, Error) << "Failed to release parameters buffers";
+
+ if (stat_->releaseBuffers())
+ LOG(RkISP1, Error) << "Failed to release stat buffers";
+
if (video_->releaseBuffers())
- LOG(RkISP1, Error) << "Failed to release buffers";
+ LOG(RkISP1, Error) << "Failed to release video buffers";
return 0;
}
int PipelineHandlerRkISP1::start(Camera *camera)
{
+ RkISP1CameraData *data = cameraData(camera);
int ret;
+ ret = param_->streamOn();
+ if (ret) {
+ LOG(RkISP1, Error)
+ << "Failed to start parameters " << camera->name();
+ return ret;
+ }
+
+ ret = stat_->streamOn();
+ if (ret) {
+ param_->streamOff();
+ LOG(RkISP1, Error)
+ << "Failed to start statistics " << camera->name();
+ return ret;
+ }
+
ret = video_->streamOn();
- if (ret)
+ if (ret) {
+ param_->streamOff();
+ stat_->streamOff();
+
LOG(RkISP1, Error)
<< "Failed to start camera " << camera->name();
+ }
activeCamera_ = camera;
+ /* Inform IPA of stream configuration and sensor controls. */
+ std::map<unsigned int, IPAStream> streamConfig;
+ streamConfig[0] = {
+ .pixelFormat = data->stream_.configuration().pixelFormat,
+ .size = data->stream_.configuration().size,
+ };
+
+ std::map<unsigned int, V4L2ControlInfoMap> entityControls;
+ entityControls[0] = data->sensor_->controls();
+
+ data->ipa_->configure(streamConfig, entityControls);
+
return ret;
}
void PipelineHandlerRkISP1::stop(Camera *camera)
{
+ RkISP1CameraData *data = cameraData(camera);
int ret;
ret = video_->streamOff();
@@ -372,6 +593,18 @@ void PipelineHandlerRkISP1::stop(Camera *camera)
LOG(RkISP1, Warning)
<< "Failed to stop camera " << camera->name();
+ ret = stat_->streamOff();
+ if (ret)
+ LOG(RkISP1, Warning)
+ << "Failed to stop statistics " << camera->name();
+
+ ret = param_->streamOff();
+ if (ret)
+ LOG(RkISP1, Warning)
+ << "Failed to stop parameters " << camera->name();
+
+ data->timeline_.reset();
+
activeCamera_ = nullptr;
}
@@ -387,12 +620,22 @@ int PipelineHandlerRkISP1::queueRequest(Camera *camera, Request *request)
return -ENOENT;
}
- int ret = video_->queueBuffer(buffer);
- if (ret < 0)
- return ret;
-
PipelineHandler::queueRequest(camera, request);
+ data->frameInfo_[data->frame_] = request;
+
+ IPAOperationData op;
+ op.operation = RKISP1_IPA_EVENT_QUEUE_REQUEST;
+ op.data = { data->frame_ };
+ op.controls = { request->controls() };
+ data->ipa_->processEvent(op);
+
+ data->timeline_.scheduleAction(new RkISP1ActionQueueBuffer(data->frame_,
+ QueueVideo,
+ video_,
+ buffer));
+ data->frame_++;
+
return 0;
}
@@ -435,11 +678,19 @@ int PipelineHandlerRkISP1::createCamera(MediaEntity *sensor)
std::unique_ptr<RkISP1CameraData> data =
utils::make_unique<RkISP1CameraData>(this);
+ data->controlInfo_.emplace(std::piecewise_construct,
+ std::forward_as_tuple(&controls::AeEnable),
+ std::forward_as_tuple(false, true));
+
data->sensor_ = new CameraSensor(sensor);
ret = data->sensor_->init();
if (ret)
return ret;
+ ret = data->loadIPA();
+ if (ret)
+ return ret;
+
std::set<Stream *> streams{ &data->stream_ };
std::shared_ptr<Camera> camera =
Camera::create(this, sensor->name(), streams);
@@ -478,7 +729,17 @@ bool PipelineHandlerRkISP1::match(DeviceEnumerator *enumerator)
if (video_->open() < 0)
return false;
+ stat_ = V4L2VideoDevice::fromEntityName(media_, "rkisp1-statistics");
+ if (stat_->open() < 0)
+ return false;
+
+ param_ = V4L2VideoDevice::fromEntityName(media_, "rkisp1-input-params");
+ if (param_->open() < 0)
+ return false;
+
video_->bufferReady.connect(this, &PipelineHandlerRkISP1::bufferReady);
+ stat_->bufferReady.connect(this, &PipelineHandlerRkISP1::statReady);
+ param_->bufferReady.connect(this, &PipelineHandlerRkISP1::paramReady);
/* Configure default links. */
if (initLinks() < 0) {
@@ -504,13 +765,52 @@ bool PipelineHandlerRkISP1::match(DeviceEnumerator *enumerator)
* Buffer Handling
*/
+void PipelineHandlerRkISP1::tryCompleteRequest(Request *request)
+{
+ if (request->hasPendingBuffers())
+ return;
+
+ if (request->metaData().empty())
+ return;
+
+ completeRequest(activeCamera_, request);
+}
+
void PipelineHandlerRkISP1::bufferReady(Buffer *buffer)
{
ASSERT(activeCamera_);
+ RkISP1CameraData *data = cameraData(activeCamera_);
Request *request = buffer->request();
+ data->timeline_.bufferReady(buffer);
+
+ if (data->frame_ <= buffer->sequence())
+ data->frame_ = buffer->sequence() + 1;
+
completeBuffer(activeCamera_, request, buffer);
- completeRequest(activeCamera_, request);
+ tryCompleteRequest(request);
+}
+
+void PipelineHandlerRkISP1::paramReady(Buffer *buffer)
+{
+ ASSERT(activeCamera_);
+ RkISP1CameraData *data = cameraData(activeCamera_);
+
+ IPAOperationData op;
+ op.operation = RKISP1_IPA_EVENT_SIGNAL_BUFFER;
+ op.data = { RKISP1_BUFFER_PARAM, buffer->index() };
+ data->ipa_->processEvent(op);
+}
+
+void PipelineHandlerRkISP1::statReady(Buffer *buffer)
+{
+ ASSERT(activeCamera_);
+ RkISP1CameraData *data = cameraData(activeCamera_);
+
+ IPAOperationData op;
+ op.operation = RKISP1_IPA_EVENT_SIGNAL_BUFFER;
+ op.data = { RKISP1_BUFFER_STAT, buffer->index() };
+ data->ipa_->processEvent(op);
}
REGISTER_PIPELINE_HANDLER(PipelineHandlerRkISP1);
diff --git a/src/libcamera/pipeline/rkisp1/rkisp1.h b/src/libcamera/pipeline/rkisp1/rkisp1.h
new file mode 100644
index 0000000000000000..1972c8bbc1286e0e
--- /dev/null
+++ b/src/libcamera/pipeline/rkisp1/rkisp1.h
@@ -0,0 +1,78 @@
+/* SPDX-License-Identifier: LGPL-2.1-or-later */
+/*
+ * Copyright (C) 2019, Google Inc.
+ *
+ * rkisp1.h - Pipeline handler for Rockchip ISP1
+ */
+#ifndef __LIBCAMERA_RKISP1_H__
+#define __LIBCAMERA_RKISP1_H__
+
+#include <ipa/rkisp1.h>
+#include <libcamera/buffer.h>
+
+#include "camera_sensor.h"
+#include "timeline.h"
+#include "v4l2_videodevice.h"
+
+namespace libcamera {
+
+enum RkISP1ActionType {
+ SetSensor,
+ SOE,
+ QueueVideo,
+ QueueParameters,
+ QueueStatistics,
+};
+
+class RkISP1ActionSetSensor : public FrameAction
+{
+public:
+ RkISP1ActionSetSensor(unsigned int frame, CameraSensor *sensor, V4L2ControlList controls)
+ : FrameAction(SetSensor, frame), sensor_(sensor), controls_(controls) {}
+
+protected:
+ void run() override;
+
+private:
+ CameraSensor *sensor_;
+ V4L2ControlList controls_;
+};
+
+class RkISP1ActionQueueBuffer : public FrameAction
+{
+public:
+ RkISP1ActionQueueBuffer(unsigned int frame, RkISP1ActionType type,
+ V4L2VideoDevice *device, Buffer *buffer)
+ : FrameAction(type, frame), device_(device), buffer_(buffer)
+ {
+ }
+
+protected:
+ void run() override;
+
+private:
+ V4L2VideoDevice *device_;
+ Buffer *buffer_;
+};
+
+class RkISP1Timeline : public Timeline
+{
+public:
+ RkISP1Timeline()
+ : Timeline()
+ {
+ setDelay(SetSensor, -1, 5);
+ setDelay(SOE, 0, -1);
+ setDelay(QueueVideo, -1, 10);
+ setDelay(QueueParameters, -1, 8);
+ setDelay(QueueStatistics, -1, 8);
+ }
+
+ void bufferReady(Buffer *buffer);
+
+ void setDelay(unsigned int type, int frame, int msdelay);
+};
+
+} /* namespace libcamera */
+
+#endif /* __LIBCAMERA_RKISP1_H__ */
diff --git a/src/libcamera/pipeline/rkisp1/timeline.cpp b/src/libcamera/pipeline/rkisp1/timeline.cpp
new file mode 100644
index 0000000000000000..ca0cd96711d84244
--- /dev/null
+++ b/src/libcamera/pipeline/rkisp1/timeline.cpp
@@ -0,0 +1,56 @@
+/* SPDX-License-Identifier: LGPL-2.1-or-later */
+/*
+ * Copyright (C) 2019, Google Inc.
+ *
+ * timeline.cpp - Timeline handler for Rockchip ISP1
+ */
+
+#include "rkisp1.h"
+
+#include "log.h"
+
+namespace libcamera {
+
+LOG_DECLARE_CATEGORY(RkISP1)
+
+void RkISP1ActionSetSensor::run()
+{
+ sensor_->setControls(&controls_);
+}
+
+void RkISP1ActionQueueBuffer::run()
+{
+ int ret = device_->queueBuffer(buffer_);
+ if (ret < 0)
+ LOG(RkISP1, Error) << "Failed to queue buffer";
+}
+
+void RkISP1Timeline::bufferReady(Buffer *buffer)
+{
+ /*
+ * Calculate SOE by taking the end of DMA set by the kernel and applying
+ * the time offsets provieded by the IPA to find the best estimate of
+ * SOE.
+ *
+ * NOTE: Make sure the IPA do not set a frame offset for the SOE action
+ * type as the frame interval is calculated using the interval between
+ * two SOE events. So using a frame interval in the SOE estimate creates
+ * a recursion.
+ */
+
+ ASSERT(frameOffset(SOE) == 0);
+
+ utils::time_point soe = std::chrono::time_point<utils::clock>()
+ + std::chrono::nanoseconds(buffer->timestamp())
+ + timeOffset(SOE);
+
+ notifyStartOfExposure(buffer->sequence(), soe);
+}
+
+void RkISP1Timeline::setDelay(unsigned int type, int frame, int msdelay)
+{
+ utils::duration delay = std::chrono::milliseconds(msdelay);
+ setRawDelay(type, frame, delay);
+}
+
+} /* namespace libcamera */
--
2.23.0
More information about the libcamera-devel
mailing list