[PATCH 05/10] mali-c55: Plumb the IPA module in
Daniel Scally
dan.scally at ideasonboard.com
Thu Jun 13 15:25:57 CEST 2024
From: Jacopo Mondi <jacopo.mondi at ideasonboard.com>
Plumb the Pipeline-IPA loop in.
Load the IPA module at camera creation time and create the loop between
the pipeline and the IPA.
When a new Request is queued the IPA is asked to prepare the parameters
buffer, once ready it notifies the pipeline which queues the parameters
to the ISP along with a buffer for statistics and frames,
Once statistics are ready they get passed to the IPA which upates its
settings for the next frame.
Acked-by: Nayden Kanchev <nayden.kanchev at arm.com>
Signed-off-by: Jacopo Mondi <jacopo.mondi at ideasonboard.com>
---
src/libcamera/pipeline/mali-c55/mali-c55.cpp | 353 +++++++++++++++++--
1 file changed, 314 insertions(+), 39 deletions(-)
diff --git a/src/libcamera/pipeline/mali-c55/mali-c55.cpp b/src/libcamera/pipeline/mali-c55/mali-c55.cpp
index b9be1207..871674cb 100644
--- a/src/libcamera/pipeline/mali-c55/mali-c55.cpp
+++ b/src/libcamera/pipeline/mali-c55/mali-c55.cpp
@@ -23,13 +23,19 @@
#include <libcamera/geometry.h>
#include <libcamera/stream.h>
+#include <libcamera/ipa/mali-c55_ipa_interface.h>
+#include <libcamera/ipa/mali-c55_ipa_proxy.h>
+
#include "libcamera/internal/bayer_format.h"
#include "libcamera/internal/camera.h"
#include "libcamera/internal/camera_sensor.h"
+#include "libcamera/internal/delayed_controls.h"
#include "libcamera/internal/device_enumerator.h"
#include "libcamera/internal/framebuffer.h"
+#include "libcamera/internal/ipa_manager.h"
#include "libcamera/internal/media_device.h"
#include "libcamera/internal/pipeline_handler.h"
+#include "libcamera/internal/request.h"
#include "libcamera/internal/v4l2_subdevice.h"
#include "libcamera/internal/v4l2_videodevice.h"
@@ -85,6 +91,16 @@ constexpr Size kMaliC55MinSize = { 128, 128 };
constexpr Size kMaliC55MaxSize = { 8192, 8192 };
constexpr unsigned int kMaliC55ISPInternalFormat = MEDIA_BUS_FMT_RGB121212_1X36;
+struct MaliC55FrameInfo {
+ Request *request;
+
+ FrameBuffer *paramBuffer;
+ FrameBuffer *statBuffer;
+
+ bool paramsDone;
+ bool statsDone;
+};
+
class MaliC55CameraData : public Camera::Private
{
public:
@@ -94,6 +110,7 @@ public:
}
int init();
+ int loadIPA();
/* Deflect these functionalities to either TPG or CameraSensor. */
const std::vector<unsigned int> mbusCodes() const;
@@ -102,7 +119,7 @@ public:
PixelFormat bestRawFormat() const;
- void updateControls();
+ void updateControls(const ControlInfoMap &ipaControls);
PixelFormat adjustRawFormat(const PixelFormat &pixFmt) const;
Size adjustRawSizes(const PixelFormat &pixFmt, const Size &rawSize) const;
@@ -115,8 +132,15 @@ public:
Stream frStream_;
Stream dsStream_;
+ std::unique_ptr<ipa::mali_c55::IPAProxyMaliC55> ipa_;
+ std::vector<IPABuffer> ipaStatBuffers_;
+ std::vector<IPABuffer> ipaParamBuffers_;
+
+ std::unique_ptr<DelayedControls> delayedCtrls_;
+
private:
void initTPGData();
+ void setSensorControls(const ControlList &sensorControls);
std::string id_;
std::vector<unsigned int> tpgCodes_;
@@ -181,6 +205,11 @@ void MaliC55CameraData::initTPGData()
tpgResolution_ = tpgSizes_.back();
}
+void MaliC55CameraData::setSensorControls(const ControlList &sensorControls)
+{
+ delayedCtrls_->push(sensorControls);
+}
+
const std::vector<unsigned int> MaliC55CameraData::mbusCodes() const
{
if (sensor_)
@@ -249,7 +278,7 @@ PixelFormat MaliC55CameraData::bestRawFormat() const
return rawFormat;
}
-void MaliC55CameraData::updateControls()
+void MaliC55CameraData::updateControls(const ControlInfoMap &ipaControls)
{
if (!sensor_)
return;
@@ -267,6 +296,9 @@ void MaliC55CameraData::updateControls()
ControlInfo(ispMinCrop, sensorInfo.analogCrop,
sensorInfo.analogCrop);
+ for (auto const &c : ipaControls)
+ controls.emplace(c.first, c.second);
+
controlInfo_ = ControlInfoMap(std::move(controls), controls::controls);
}
@@ -321,6 +353,46 @@ Size MaliC55CameraData::adjustRawSizes(const PixelFormat &rawFmt, const Size &ra
return bestSize;
}
+int MaliC55CameraData::loadIPA()
+{
+ int ret;
+
+ ipa_ = IPAManager::createIPA<ipa::mali_c55::IPAProxyMaliC55>(pipe(), 1, 1);
+ if (!ipa_)
+ return -ENOENT;
+
+ ipa_->setSensorControls.connect(this, &MaliC55CameraData::setSensorControls);
+
+ /* Do not initialize IPA for TPG. */
+ if (!sensor_)
+ return 0;
+
+ std::string ipaTuningFile = ipa_->configurationFile(sensor_->model() + ".yaml");
+ if (ipaTuningFile.empty())
+ ipaTuningFile = ipa_->configurationFile("uncalibrated.yaml");
+
+ /* We need to inform the IPA of the sensor configuration */
+ ipa::mali_c55::IPAConfigInfo ipaConfig{};
+
+ ret = sensor_->sensorInfo(&ipaConfig.sensorInfo);
+ if (ret)
+ return ret;
+
+ ipaConfig.sensorControls = sensor_->controls();
+
+ ControlInfoMap ipaControls;
+ ret = ipa_->init({ ipaTuningFile, sensor_->model() }, ipaConfig,
+ &ipaControls);
+ if (ret) {
+ LOG(MaliC55, Error) << "Failed to initialise the Mali-C55 IPA";
+ return ret;
+ }
+
+ updateControls(ipaControls);
+
+ return 0;
+}
+
class MaliC55CameraConfiguration : public CameraConfiguration
{
public:
@@ -330,6 +402,7 @@ public:
}
Status validate() override;
+ const Transform &combinedTransform() { return combinedTransform_; }
V4L2SubdeviceFormat sensorFormat_;
@@ -337,6 +410,7 @@ private:
static constexpr unsigned int kMaxStreams = 2;
const MaliC55CameraData *data_;
+ Transform combinedTransform_;
};
CameraConfiguration::Status MaliC55CameraConfiguration::validate()
@@ -346,6 +420,11 @@ CameraConfiguration::Status MaliC55CameraConfiguration::validate()
if (config_.empty())
return Invalid;
+ Orientation requestedOrientation = orientation;
+ combinedTransform_ = data_->sensor_->computeTransform(&orientation);
+ if (orientation != requestedOrientation)
+ status = Adjusted;
+
/* Only 2 streams available. */
if (config_.size() > kMaxStreams) {
config_.resize(kMaxStreams);
@@ -492,7 +571,10 @@ public:
int queueRequestDevice(Camera *camera, Request *request) override;
void bufferReady(FrameBuffer *buffer);
+ void paramsBufferReady(FrameBuffer *buffer);
void statsBufferReady(FrameBuffer *buffer);
+ void paramsComputed(unsigned int requestId);
+ void statsProcessed(unsigned int requestId, const ControlList &metadata);
bool match(DeviceEnumerator *enumerator) override;
@@ -536,6 +618,10 @@ private:
pipe.stream = nullptr;
}
+ MaliC55FrameInfo *findFrameInfo(FrameBuffer *buffer);
+ MaliC55FrameInfo *findFrameInfo(Request *request);
+ void tryComplete(MaliC55FrameInfo *info);
+
int configureRawStream(MaliC55CameraData *data,
const StreamConfiguration &config,
V4L2SubdeviceFormat &subdevFormat);
@@ -545,7 +631,7 @@ private:
void applyScalerCrop(Camera *camera, const ControlList &controls);
- void registerMaliCamera(std::unique_ptr<MaliC55CameraData> data,
+ bool registerMaliCamera(std::unique_ptr<MaliC55CameraData> data,
const std::string &name);
bool registerTPGCamera(MediaLink *link);
bool registerSensorCamera(MediaLink *link);
@@ -561,6 +647,8 @@ private:
std::vector<std::unique_ptr<FrameBuffer>> paramsBuffers_;
std::queue<FrameBuffer *> availableParamsBuffers_;
+ std::map<unsigned int, MaliC55FrameInfo> frameInfoMap_;
+
std::array<MaliC55Pipe, MaliC55NumPipes> pipes_;
bool dsFitted_;
@@ -806,6 +894,11 @@ int PipelineHandlerMaliC55::configure(Camera *camera,
if (ret)
return ret;
+ ret = data->sensor_->setFormat(&subdevFormat,
+ maliConfig->combinedTransform());
+ if (ret)
+ return ret;
+
if (data->csi_) {
ret = data->csi_->setFormat(0, &subdevFormat);
if (ret)
@@ -876,7 +969,31 @@ int PipelineHandlerMaliC55::configure(Camera *camera,
pipe->stream = stream;
}
- data->updateControls();
+ /* We need to inform the IPA of the sensor configuration */
+ ipa::mali_c55::IPAConfigInfo ipaConfig{};
+
+ ret = data->sensor_->sensorInfo(&ipaConfig.sensorInfo);
+ if (ret)
+ return ret;
+
+ ipaConfig.sensorControls = data->sensor_->controls();
+
+ /*
+ * And we also need to tell the IPA the bayerOrder of the data (as
+ * affected by any flips that we've configured)
+ */
+ const Transform &combinedTransform = maliConfig->combinedTransform();
+ BayerFormat::Order bayerOrder = data->sensor_->bayerOrder(combinedTransform);
+
+ ControlInfoMap ipaControls;
+ ret = data->ipa_->configure(ipaConfig, utils::to_underlying(bayerOrder),
+ &ipaControls);
+ if (ret) {
+ LOG(MaliC55, Error) << "Failed to configure IPA";
+ return ret;
+ }
+
+ data->updateControls(ipaControls);
return 0;
}
@@ -890,8 +1007,10 @@ int PipelineHandlerMaliC55::exportFrameBuffers(Camera *camera, Stream *stream,
return pipe->cap->exportBuffers(count, buffers);
}
-void PipelineHandlerMaliC55::freeBuffers([[maybe_unused]] Camera *camera)
+void PipelineHandlerMaliC55::freeBuffers(Camera *camera)
{
+ MaliC55CameraData *data = cameraData(camera);
+
while (!availableStatsBuffers_.empty())
availableStatsBuffers_.pop();
while (!availableParamsBuffers_.empty())
@@ -900,11 +1019,16 @@ void PipelineHandlerMaliC55::freeBuffers([[maybe_unused]] Camera *camera)
statsBuffers_.clear();
paramsBuffers_.clear();
+ data->ipa_->unmapBuffers(data->ipaStatBuffers_);
+ data->ipaStatBuffers_.clear();
+ data->ipa_->unmapBuffers(data->ipaParamBuffers_);
+ data->ipaParamBuffers_.clear();
+
if (stats_->releaseBuffers())
LOG(MaliC55, Error) << "Failed to release stats buffers";
if (params_->releaseBuffers())
- LOG(MaliC55, Error) << "Failed to release stats buffers";
+ LOG(MaliC55, Error) << "Failed to release params buffers";
return;
}
@@ -912,6 +1036,7 @@ void PipelineHandlerMaliC55::freeBuffers([[maybe_unused]] Camera *camera)
int PipelineHandlerMaliC55::allocateBuffers(Camera *camera)
{
MaliC55CameraData *data = cameraData(camera);
+ unsigned int ipaBufferId = 1;
unsigned int bufferCount;
int ret;
@@ -924,27 +1049,47 @@ int PipelineHandlerMaliC55::allocateBuffers(Camera *camera)
if (ret < 0)
return ret;
- for (std::unique_ptr<FrameBuffer> &buffer : statsBuffers_)
+ for (std::unique_ptr<FrameBuffer> &buffer : statsBuffers_) {
+ buffer->setCookie(ipaBufferId++);
+ data->ipaStatBuffers_.emplace_back(buffer->cookie(),
+ buffer->planes());
availableStatsBuffers_.push(buffer.get());
+ }
+
+ data->ipa_->mapBuffers(data->ipaStatBuffers_, true);
ret = params_->allocateBuffers(bufferCount, ¶msBuffers_);
if (ret < 0)
return ret;
- for (std::unique_ptr<FrameBuffer> &buffer : paramsBuffers_)
+ for (std::unique_ptr<FrameBuffer> &buffer : paramsBuffers_) {
+ buffer->setCookie(ipaBufferId++);
+ data->ipaParamBuffers_.emplace_back(buffer->cookie(),
+ buffer->planes());
availableParamsBuffers_.push(buffer.get());
+ }
+
+ data->ipa_->mapBuffers(data->ipaParamBuffers_, false);
return 0;
}
int PipelineHandlerMaliC55::start(Camera *camera, [[maybe_unused]] const ControlList *controls)
{
+ MaliC55CameraData *data = cameraData(camera);
int ret;
ret = allocateBuffers(camera);
if (ret)
return ret;
+ ret = data->ipa_->start();
+ if (ret) {
+ LOG(MaliC55, Error) << "Failed to start IPA" << camera->id();
+ freeBuffers(camera);
+ return ret;
+ }
+
for (MaliC55Pipe &pipe : pipes_) {
if (!pipe.stream)
continue;
@@ -954,6 +1099,7 @@ int PipelineHandlerMaliC55::start(Camera *camera, [[maybe_unused]] const Control
ret = pipe.cap->importBuffers(stream->configuration().bufferCount);
if (ret) {
LOG(MaliC55, Error) << "Failed to import buffers";
+ data->ipa_->stop();
freeBuffers(camera);
return ret;
}
@@ -961,6 +1107,7 @@ int PipelineHandlerMaliC55::start(Camera *camera, [[maybe_unused]] const Control
ret = pipe.cap->streamOn();
if (ret) {
LOG(MaliC55, Error) << "Failed to start stream";
+ data->ipa_->stop();
freeBuffers(camera);
return ret;
}
@@ -970,6 +1117,8 @@ int PipelineHandlerMaliC55::start(Camera *camera, [[maybe_unused]] const Control
if (ret) {
LOG(MaliC55, Error) << "Failed to start stats stream";
+ data->ipa_->stop();
+
for (MaliC55Pipe &pipe : pipes_) {
if (pipe.stream)
pipe.cap->streamOff();
@@ -984,6 +1133,7 @@ int PipelineHandlerMaliC55::start(Camera *camera, [[maybe_unused]] const Control
LOG(MaliC55, Error) << "Failed to start params stream";
stats_->streamOff();
+ data->ipa_->stop();
for (MaliC55Pipe &pipe : pipes_) {
if (pipe.stream)
@@ -994,11 +1144,19 @@ int PipelineHandlerMaliC55::start(Camera *camera, [[maybe_unused]] const Control
return ret;
}
+ ret = isp_->setFrameStartEnabled(true);
+ if (ret)
+ LOG(MaliC55, Error) << "Failed to enable frame start events";
+
return 0;
}
-void PipelineHandlerMaliC55::stopDevice([[maybe_unused]] Camera *camera)
+void PipelineHandlerMaliC55::stopDevice(Camera *camera)
{
+ MaliC55CameraData *data = cameraData(camera);
+
+ isp_->setFrameStartEnabled(false);
+
for (MaliC55Pipe &pipe : pipes_) {
if (!pipe.stream)
continue;
@@ -1009,6 +1167,7 @@ void PipelineHandlerMaliC55::stopDevice([[maybe_unused]] Camera *camera)
stats_->streamOff();
params_->streamOff();
+ data->ipa_->stop();
freeBuffers(camera);
}
@@ -1112,30 +1271,16 @@ void PipelineHandlerMaliC55::applyScalerCrop(Camera *camera,
int PipelineHandlerMaliC55::queueRequestDevice(Camera *camera, Request *request)
{
- FrameBuffer *statsBuffer;
- int ret;
+ MaliC55CameraData *data = cameraData(camera);
if (availableStatsBuffers_.empty()) {
LOG(MaliC55, Error) << "Stats buffer underrun";
return -ENOENT;
}
- statsBuffer = availableStatsBuffers_.front();
- availableStatsBuffers_.pop();
-
- /*
- * We need to associate the Request to this buffer even though it's a
- * purely internal one because we will need to use request->sequence()
- * later.
- */
- statsBuffer->_d()->setRequest(request);
-
- for (auto &[stream, buffer] : request->buffers()) {
- MaliC55Pipe *pipe = pipeFromStream(cameraData(camera), stream);
-
- ret = pipe->cap->queueBuffer(buffer);
- if (ret)
- return ret;
+ if (availableParamsBuffers_.empty()) {
+ LOG(MaliC55, Error) << "Params buffer underrun";
+ return -ENOENT;
}
/*
@@ -1147,29 +1292,143 @@ int PipelineHandlerMaliC55::queueRequestDevice(Camera *camera, Request *request)
*/
applyScalerCrop(camera, request->controls());
- ret = stats_->queueBuffer(statsBuffer);
- if (ret)
- return ret;
+ MaliC55FrameInfo frameInfo;
+ frameInfo.request = request;
+
+ frameInfo.statBuffer = availableStatsBuffers_.front();
+ availableStatsBuffers_.pop();
+ frameInfo.paramBuffer = availableParamsBuffers_.front();
+ availableParamsBuffers_.pop();
+
+ frameInfo.paramsDone = false;
+ frameInfo.statsDone = false;
+
+ frameInfoMap_[request->sequence()] = frameInfo;
+
+ data->ipa_->queueRequest(request->sequence(), request->controls());
+ data->ipa_->fillParams(request->sequence(),
+ frameInfo.paramBuffer->cookie());
return 0;
}
+MaliC55FrameInfo *PipelineHandlerMaliC55::findFrameInfo(Request *request)
+{
+ for (auto &[sequence, info] : frameInfoMap_) {
+ if (info.request == request)
+ return &info;
+ }
+
+ return nullptr;
+}
+
+MaliC55FrameInfo *PipelineHandlerMaliC55::findFrameInfo(FrameBuffer *buffer)
+{
+ for (auto &[sequence, info] : frameInfoMap_) {
+ if (info.paramBuffer == buffer ||
+ info.statBuffer == buffer)
+ return &info;
+ }
+
+ return nullptr;
+}
+
+void PipelineHandlerMaliC55::tryComplete(MaliC55FrameInfo *info)
+{
+ if (!info->paramsDone)
+ return;
+ if (!info->statsDone)
+ return;
+
+ Request *request = info->request;
+ if (request->hasPendingBuffers())
+ return;
+
+ availableStatsBuffers_.push(info->statBuffer);
+ availableParamsBuffers_.push(info->paramBuffer);
+
+ frameInfoMap_.erase(request->sequence());
+
+ completeRequest(request);
+}
+
void PipelineHandlerMaliC55::bufferReady(FrameBuffer *buffer)
{
Request *request = buffer->request();
+ MaliC55FrameInfo *info = findFrameInfo(request);
+ ASSERT(info);
if (completeBuffer(request, buffer))
- completeRequest(request);
+ tryComplete(info);
+}
+
+void PipelineHandlerMaliC55::paramsBufferReady(FrameBuffer *buffer)
+{
+ MaliC55FrameInfo *info = findFrameInfo(buffer);
+ ASSERT(info);
+
+ info->paramsDone = true;
+
+ tryComplete(info);
}
void PipelineHandlerMaliC55::statsBufferReady(FrameBuffer *buffer)
{
- availableStatsBuffers_.push(buffer);
+ MaliC55FrameInfo *info = findFrameInfo(buffer);
+ ASSERT(info);
+
+ Request *request = info->request;
+ MaliC55CameraData *data = cameraData(request->_d()->camera());
+
+ ControlList sensorControls = data->delayedCtrls_->get(buffer->metadata().sequence);
+
+ data->ipa_->processStats(request->sequence(), buffer->cookie(),
+ sensorControls);
+}
+
+void PipelineHandlerMaliC55::paramsComputed(unsigned int requestId)
+{
+ MaliC55FrameInfo &frameInfo = frameInfoMap_[requestId];
+ Request *request = frameInfo.request;
+ MaliC55CameraData *data = cameraData(request->_d()->camera());
+
+ /*
+ * Queue buffers for stats and params, then queue buffers to the capture
+ * video devices.
+ */
+
+ frameInfo.paramBuffer->_d()->metadata().planes()[0].bytesused =
+ sizeof(struct mali_c55_params_buffer);
+ params_->queueBuffer(frameInfo.paramBuffer);
+ stats_->queueBuffer(frameInfo.statBuffer);
+
+ for (auto &[stream, buffer] : request->buffers()) {
+ MaliC55Pipe *pipe = pipeFromStream(data, stream);
+
+ pipe->cap->queueBuffer(buffer);
+ }
}
-void PipelineHandlerMaliC55::registerMaliCamera(std::unique_ptr<MaliC55CameraData> data,
+void PipelineHandlerMaliC55::statsProcessed(unsigned int requestId,
+ const ControlList &metadata)
+{
+ MaliC55FrameInfo &frameInfo = frameInfoMap_[requestId];
+
+ frameInfo.statsDone = true;
+ frameInfo.request->metadata().merge(metadata);
+
+ tryComplete(&frameInfo);
+}
+
+bool PipelineHandlerMaliC55::registerMaliCamera(std::unique_ptr<MaliC55CameraData> data,
const std::string &name)
{
+ if (data->loadIPA())
+ return false;
+
+ data->ipa_->statsProcessed.connect(this, &PipelineHandlerMaliC55::statsProcessed);
+ data->ipa_->paramsComputed.connect(this, &PipelineHandlerMaliC55::paramsComputed);
+
std::set<Stream *> streams{ &data->frStream_ };
if (dsFitted_)
streams.insert(&data->dsStream_);
@@ -1177,6 +1436,8 @@ void PipelineHandlerMaliC55::registerMaliCamera(std::unique_ptr<MaliC55CameraDat
std::shared_ptr<Camera> camera = Camera::create(std::move(data),
name, streams);
registerCamera(std::move(camera));
+
+ return true;
}
/*
@@ -1202,9 +1463,7 @@ bool PipelineHandlerMaliC55::registerTPGCamera(MediaLink *link)
if (data->init())
return false;
- registerMaliCamera(std::move(data), name);
-
- return true;
+ return registerMaliCamera(std::move(data), name);
}
/*
@@ -1230,11 +1489,26 @@ bool PipelineHandlerMaliC55::registerSensorCamera(MediaLink *ispLink)
if (data->init())
return false;
- /* \todo: Init properties. */
+ /*
+ * \todo Read delay values from the sensor itself or from a
+ * a sensor database. For now use generic values taken from
+ * the Raspberry Pi and listed as 'generic values'.
+ */
+ std::unordered_map<uint32_t, DelayedControls::ControlParams> params = {
+ { V4L2_CID_ANALOGUE_GAIN, { 1, false } },
+ { V4L2_CID_EXPOSURE, { 2, false } },
+ };
+
+ data->delayedCtrls_ =
+ std::make_unique<DelayedControls>(data->sensor_->device(),
+ params);
+ isp_->frameStart.connect(data->delayedCtrls_.get(),
+ &DelayedControls::applyControls);
- data->updateControls();
+ /* \todo: Init properties. */
- registerMaliCamera(std::move(data), sensor->name());
+ if (!registerMaliCamera(std::move(data), sensor->name()))
+ return false;
}
return true;
@@ -1301,6 +1575,7 @@ bool PipelineHandlerMaliC55::match(DeviceEnumerator *enumerator)
}
stats_->bufferReady.connect(this, &PipelineHandlerMaliC55::statsBufferReady);
+ params_->bufferReady.connect(this, &PipelineHandlerMaliC55::paramsBufferReady);
ispSink = isp_->entity()->getPadByIndex(0);
if (!ispSink || ispSink->links().empty()) {
--
2.30.2
More information about the libcamera-devel
mailing list