[libcamera-devel] [PATCH v2 4/5] libcamera: ipu3: Initialize controls in the IPA
paul.elder at ideasonboard.com
paul.elder at ideasonboard.com
Tue Aug 3 12:02:38 CEST 2021
Hi Jacopo,
On Wed, Jul 28, 2021 at 06:11:15PM +0200, Jacopo Mondi wrote:
> All the IPU3 Camera controls are currently initialized by the pipeline
> handler which initializes them using the camera sensor configuration and
> platform specific requirements.
>
> However, some controls are better initialized by the IPA, which might,
> in example, cap the exposure times and frame duration to the constraints
> of its algorithms implementation.
>
> Also, moving forward, the IPA should register controls to report its
> capabilities, in example the ability to enable/disable 3A algorithms on
> request.
>
> Move the existing controls initialization to the IPA, by providing
> the sensor configuration and its controls to the IPU3IPA::init()
> function, which initializes controls and returns them to the pipeline
> through an output parameter.
>
> The existing controls initialization has been copied verbatim from the
> pipeline handler to the IPA, if not a for few line breaks adjustments
> and the resulting Camera controls values are not changed .
>
> Signed-off-by: Jacopo Mondi <jacopo at jmondi.org>
Reviewed-by: Paul Elder <paul.elder at ideasonboard.com>
> ---
> include/libcamera/ipa/ipu3.mojom | 9 ++-
> src/ipa/ipu3/ipu3.cpp | 71 +++++++++++++++++--
> src/libcamera/pipeline/ipu3/ipu3.cpp | 100 ++++++++++++---------------
> 3 files changed, 121 insertions(+), 59 deletions(-)
>
> diff --git a/include/libcamera/ipa/ipu3.mojom b/include/libcamera/ipa/ipu3.mojom
> index 911a3a072464..5fb53d0fcc4f 100644
> --- a/include/libcamera/ipa/ipu3.mojom
> +++ b/include/libcamera/ipa/ipu3.mojom
> @@ -30,6 +30,12 @@ struct IPU3Action {
> libcamera.ControlList controls;
> };
>
> +struct IPAInitInfo {
> + libcamera.IPASettings settings;
> + libcamera.IPACameraSensorInfo sensorInfo;
> + libcamera.ControlInfoMap sensorControls;
> +};
> +
> struct IPAConfigInfo {
> libcamera.IPACameraSensorInfo sensorInfo;
> map<uint32, libcamera.ControlInfoMap> entityControls;
> @@ -38,7 +44,8 @@ struct IPAConfigInfo {
> };
>
> interface IPAIPU3Interface {
> - init(libcamera.IPASettings settings) => (int32 ret);
> + init(IPAInitInfo initInfo)
> + => (int32 ret, libcamera.ControlInfoMap ipaControls);
> start() => (int32 ret);
> stop();
>
> diff --git a/src/ipa/ipu3/ipu3.cpp b/src/ipa/ipu3/ipu3.cpp
> index 71698d36e50f..7a0c89ecb5fa 100644
> --- a/src/ipa/ipu3/ipu3.cpp
> +++ b/src/ipa/ipu3/ipu3.cpp
> @@ -5,8 +5,10 @@
> * ipu3.cpp - IPU3 Image Processing Algorithms
> */
>
> +#include <array>
> #include <stdint.h>
> #include <sys/mman.h>
> +#include <utility>
>
> #include <linux/intel-ipu3.h>
> #include <linux/v4l2-controls.h>
> @@ -38,7 +40,8 @@ namespace ipa::ipu3 {
> class IPAIPU3 : public IPAIPU3Interface
> {
> public:
> - int init(const IPASettings &settings) override;
> + int init(const IPAInitInfo &initInfo, ControlInfoMap *ipaControls) override;
> +
> int start() override;
> void stop() override {}
>
> @@ -86,14 +89,74 @@ private:
> struct ipu3_uapi_grid_config bdsGrid_;
> };
>
> -int IPAIPU3::init(const IPASettings &settings)
> +/**
> + * Initialize the IPA module and its controls.
> + *
> + * This function receives the camera sensor information from the pipeline
> + * handler, computes the limits of the controls it handles and returns
> + * them in the \a ipaControls output parameter.
> + */
> +int IPAIPU3::init(const IPAInitInfo &initInfo, ControlInfoMap *ipaControls)
> {
> - camHelper_ = CameraSensorHelperFactory::create(settings.sensorModel);
> + const std::string &sensorModel = initInfo.settings.sensorModel;
> + /* Initialize the camera sensor helper. */
> + camHelper_ = CameraSensorHelperFactory::create(sensorModel);
> if (camHelper_ == nullptr) {
> - LOG(IPAIPU3, Error) << "Failed to create camera sensor helper for " << settings.sensorModel;
> + LOG(IPAIPU3, Error) << "Failed to create camera sensor helper for "
> + << sensorModel;
> return -ENODEV;
> }
>
> + /* Initialize Controls. */
> + const ControlInfoMap &sensorControls = initInfo.sensorControls;
> + ControlInfoMap::Map controls{};
> +
> + /*
> + * Compute exposure time limits.
> + *
> + * Initialize the control using the line length and pixel rate of the
> + * current configuration converted to microseconds. Use the
> + * V4L2_CID_EXPOSURE control to get exposure min, max and default and
> + * convert it from lines to microseconds.
> + */
> + const IPACameraSensorInfo &sensorInfo = initInfo.sensorInfo;
> + double lineDuration = sensorInfo.lineLength / (sensorInfo.pixelRate / 1e6);
> + const ControlInfo &v4l2Exposure = sensorControls.find(V4L2_CID_EXPOSURE)->second;
> + int32_t minExposure = v4l2Exposure.min().get<int32_t>() * lineDuration;
> + int32_t maxExposure = v4l2Exposure.max().get<int32_t>() * lineDuration;
> + int32_t defExposure = v4l2Exposure.def().get<int32_t>() * lineDuration;
> + controls[&controls::ExposureTime] = ControlInfo(minExposure, maxExposure,
> + defExposure);
> +
> + /*
> + * Compute the frame duration limits.
> + *
> + * The frame length is computed assuming a fixed line length combined
> + * with the vertical frame sizes.
> + */
> + const ControlInfo &v4l2HBlank = sensorControls.find(V4L2_CID_HBLANK)->second;
> + uint32_t hblank = v4l2HBlank.def().get<int32_t>();
> + uint32_t lineLength = sensorInfo.outputSize.width + hblank;
> +
> + const ControlInfo &v4l2VBlank = sensorControls.find(V4L2_CID_VBLANK)->second;
> + std::array<uint32_t, 3> frameHeights{
> + v4l2VBlank.min().get<int32_t>() + sensorInfo.outputSize.height,
> + v4l2VBlank.max().get<int32_t>() + sensorInfo.outputSize.height,
> + v4l2VBlank.def().get<int32_t>() + sensorInfo.outputSize.height,
> + };
> +
> + std::array<int64_t, 3> frameDurations;
> + for (unsigned int i = 0; i < frameHeights.size(); ++i) {
> + uint64_t frameSize = lineLength * frameHeights[i];
> + frameDurations[i] = frameSize / (sensorInfo.pixelRate / 1000000U);
> + }
> +
> + controls[&controls::FrameDurationLimits] = ControlInfo(frameDurations[0],
> + frameDurations[1],
> + frameDurations[2]);
> +
> + *ipaControls = ControlInfoMap(std::move(controls), controls::controls);
> +
> return 0;
> }
>
> diff --git a/src/libcamera/pipeline/ipu3/ipu3.cpp b/src/libcamera/pipeline/ipu3/ipu3.cpp
> index 048993365b44..91fc1f7dc9b7 100644
> --- a/src/libcamera/pipeline/ipu3/ipu3.cpp
> +++ b/src/libcamera/pipeline/ipu3/ipu3.cpp
> @@ -88,6 +88,8 @@ public:
>
> std::queue<Request *> pendingRequests_;
>
> + ControlInfoMap ipaControls_;
> +
> private:
> void queueFrameAction(unsigned int id,
> const ipa::ipu3::IPU3Action &action);
> @@ -940,7 +942,6 @@ int PipelineHandlerIPU3::initControls(IPU3CameraData *data)
> return ret;
>
> ControlInfoMap::Map controls = IPU3Controls;
> - const ControlInfoMap &sensorControls = sensor->controls();
> const std::vector<int32_t> &testPatternModes = sensor->testPatternModes();
> if (!testPatternModes.empty()) {
> std::vector<ControlValue> values;
> @@ -952,58 +953,6 @@ int PipelineHandlerIPU3::initControls(IPU3CameraData *data)
> controls[&controls::draft::TestPatternMode] = ControlInfo(values);
> }
>
> - /*
> - * Compute exposure time limits.
> - *
> - * Initialize the control using the line length and pixel rate of the
> - * current configuration converted to microseconds. Use the
> - * V4L2_CID_EXPOSURE control to get exposure min, max and default and
> - * convert it from lines to microseconds.
> - */
> - double lineDuration = sensorInfo.lineLength
> - / (sensorInfo.pixelRate / 1e6);
> - const ControlInfo &v4l2Exposure = sensorControls.find(V4L2_CID_EXPOSURE)->second;
> - int32_t minExposure = v4l2Exposure.min().get<int32_t>() * lineDuration;
> - int32_t maxExposure = v4l2Exposure.max().get<int32_t>() * lineDuration;
> - int32_t defExposure = v4l2Exposure.def().get<int32_t>() * lineDuration;
> -
> - /*
> - * \todo Report the actual exposure time, use the default for the
> - * moment.
> - */
> - data->exposureTime_ = defExposure;
> -
> - controls[&controls::ExposureTime] = ControlInfo(minExposure, maxExposure,
> - defExposure);
> -
> - /*
> - * Compute the frame duration limits.
> - *
> - * The frame length is computed assuming a fixed line length combined
> - * with the vertical frame sizes.
> - */
> - const ControlInfo &v4l2HBlank = sensorControls.find(V4L2_CID_HBLANK)->second;
> - uint32_t hblank = v4l2HBlank.def().get<int32_t>();
> - uint32_t lineLength = sensorInfo.outputSize.width + hblank;
> -
> - const ControlInfo &v4l2VBlank = sensorControls.find(V4L2_CID_VBLANK)->second;
> - std::array<uint32_t, 3> frameHeights{
> - v4l2VBlank.min().get<int32_t>() + sensorInfo.outputSize.height,
> - v4l2VBlank.max().get<int32_t>() + sensorInfo.outputSize.height,
> - v4l2VBlank.def().get<int32_t>() + sensorInfo.outputSize.height,
> - };
> -
> - std::array<int64_t, 3> frameDurations;
> - for (unsigned int i = 0; i < frameHeights.size(); ++i) {
> - uint64_t frameSize = lineLength * frameHeights[i];
> - frameDurations[i] = frameSize / (sensorInfo.pixelRate / 1000000U);
> - }
> -
> - controls[&controls::FrameDurationLimits] =
> - ControlInfo(frameDurations[0],
> - frameDurations[1],
> - frameDurations[2]);
> -
> /*
> * Compute the scaler crop limits.
> *
> @@ -1057,9 +1006,14 @@ int PipelineHandlerIPU3::initControls(IPU3CameraData *data)
>
> controls[&controls::ScalerCrop] = ControlInfo(minCrop, maxCrop, maxCrop);
>
> + /* Add the IPA registered controls to list of camera controls. */
> + for (const auto &ipaControl : data->ipaControls_)
> + controls[ipaControl.first] = ipaControl.second;
> +
> data->controlInfo_ = ControlInfoMap(std::move(controls),
> controls::controls);
>
> +
> return 0;
> }
>
> @@ -1209,13 +1163,51 @@ int IPU3CameraData::loadIPA()
>
> ipa_->queueFrameAction.connect(this, &IPU3CameraData::queueFrameAction);
>
> + /*
> + * Pass the sensor info to the IPA to initialize controls.
> + *
> + * \todo Find a way to initialize IPA controls without basing their
> + * limits on a particular sensor mode. We currently pass sensor
> + * information corresponding to the largest sensor resolution, and the
> + * IPA uses this to compute limits for supported controls. There's a
> + * discrepancy between the need to compute IPA control limits at init
> + * time, and the fact that those limits may depend on the sensor mode.
> + * Research is required to find out to handle this issue.
> + */
> CameraSensor *sensor = cio2_.sensor();
> - int ret = ipa_->init(IPASettings{ "", sensor->model() });
> + V4L2SubdeviceFormat sensorFormat = {};
> + sensorFormat.size = sensor->resolution();
> + int ret = sensor->setFormat(&sensorFormat);
> + if (ret)
> + return ret;
> +
> + IPACameraSensorInfo sensorInfo{};
> + ret = sensor->sensorInfo(&sensorInfo);
> + if (ret)
> + return ret;
> +
> + ipa::ipu3::IPAInitInfo initInfo{
> + { "", sensor->model() },
> + sensorInfo,
> + sensor->controls(),
> + };
> + ret = ipa_->init(initInfo, &ipaControls_);
> if (ret) {
> LOG(IPU3, Error) << "Failed to initialise the IPU3 IPA";
> return ret;
> }
>
> + /*
> + * \todo Report the actual exposure time, use the default for the
> + * moment.
> + */
> + const auto exposureInfo = ipaControls_.find(&controls::ExposureTime);
> + if (exposureInfo == ipaControls_.end()) {
> + LOG(IPU3, Error) << "Exposure control not initializaed by the IPA";
> + return -EINVAL;
> + }
> + exposureTime_ = exposureInfo->second.def().get<int32_t>();
> +
> return 0;
> }
>
> --
> 2.32.0
>
More information about the libcamera-devel
mailing list