[libcamera-devel] [PATCH v4 06/11] libcamera: ipa: ipu3: Add an IPA skeleton for the IPU3 pipeline
Laurent Pinchart
laurent.pinchart at ideasonboard.com
Fri Feb 5 00:30:16 CET 2021
Hi Niklas,
Thank you for the patch.
On Fri, Feb 05, 2021 at 12:26:08AM +0100, Niklas Söderlund wrote:
> Add an empty IPA skeleton for the IPU3 pipeline. The skeleton IPA
> handles the flow of parameter and statistic buffers but does not read or
> write anything in the buffers. It also allows the IPA to set sensor
> controls but does not implement any logic to set optimal values and
> instead sets the V4L2 exposure and gain controls to max and keeps them
> at that setting.
>
> This IPA is meant as a base to allow the pipeline handler to be wired up
> to an IPA. The image algorithms can then later be added to the IPA
> independently from also having to add plumbing to the pipeline handler.
>
> Signed-off-by: Niklas Söderlund <niklas.soderlund at ragnatech.se>
Reviewed-by: Laurent Pinchart <laurent.pinchart at ideasonboard.com>
> ---
> * Changes since v1
> - Updated commit message.
> - Rename IPU3_IPA_EVENT_PARSE_STAT to IPU3_IPA_EVENT_PARSE_STAT.
> - Sort and drop unused headers.
> - Fix style issue in meson.build and add dependency on libatomic.
> - Switch to MappedFrameBuffer interface.
> - Add result of IPA configuration status.
>
> * Changes since v2
> - Include stdint.h
> - s/std::max<uint32_t>/std::max/
> - Erase iterator instead of id in unmapBuffers()
>
> * Changes since v3
> - Add a new interface to be able to process controls separate from
> filling the parameters buffer.
> ---
> include/libcamera/ipa/ipu3.h | 23 ++++
> src/ipa/ipu3/ipu3.cpp | 242 +++++++++++++++++++++++++++++++++++
> src/ipa/ipu3/meson.build | 21 +++
> src/ipa/meson.build | 2 +-
> 4 files changed, 287 insertions(+), 1 deletion(-)
> create mode 100644 include/libcamera/ipa/ipu3.h
> create mode 100644 src/ipa/ipu3/ipu3.cpp
> create mode 100644 src/ipa/ipu3/meson.build
>
> diff --git a/include/libcamera/ipa/ipu3.h b/include/libcamera/ipa/ipu3.h
> new file mode 100644
> index 0000000000000000..cbaaef04417b701b
> --- /dev/null
> +++ b/include/libcamera/ipa/ipu3.h
> @@ -0,0 +1,23 @@
> +/* SPDX-License-Identifier: LGPL-2.1-or-later */
> +/*
> + * Copyright (C) 2020, Google Inc.
> + *
> + * ipu3.h - Image Processing Algorithm interface for IPU3
> + */
> +#ifndef __LIBCAMERA_IPA_INTERFACE_IPU3_H__
> +#define __LIBCAMERA_IPA_INTERFACE_IPU3_H__
> +
> +#ifndef __DOXYGEN__
> +
> +enum IPU3Operations {
> + IPU3_IPA_ACTION_SET_SENSOR_CONTROLS = 1,
> + IPU3_IPA_ACTION_PARAM_FILLED = 2,
> + IPU3_IPA_ACTION_METADATA_READY = 3,
> + IPU3_IPA_EVENT_PROCESS_CONTROLS = 4,
> + IPU3_IPA_EVENT_STAT_READY = 5,
> + IPU3_IPA_EVENT_FILL_PARAMS = 6,
> +};
> +
> +#endif /* __DOXYGEN__ */
> +
> +#endif /* __LIBCAMERA_IPA_INTERFACE_IPU3_H__ */
> diff --git a/src/ipa/ipu3/ipu3.cpp b/src/ipa/ipu3/ipu3.cpp
> new file mode 100644
> index 0000000000000000..b11b03efa6ceb666
> --- /dev/null
> +++ b/src/ipa/ipu3/ipu3.cpp
> @@ -0,0 +1,242 @@
> +/* SPDX-License-Identifier: LGPL-2.1-or-later */
> +/*
> + * Copyright (C) 2020, Google Inc.
> + *
> + * ipu3.cpp - IPU3 Image Processing Algorithms
> + */
> +
> +#include <libcamera/ipa/ipu3.h>
> +
> +#include <stdint.h>
> +#include <sys/mman.h>
> +
> +#include <linux/intel-ipu3.h>
> +#include <linux/v4l2-controls.h>
> +
> +#include <libcamera/buffer.h>
> +#include <libcamera/control_ids.h>
> +#include <libcamera/ipa/ipa_interface.h>
> +#include <libcamera/ipa/ipa_module_info.h>
> +#include <libcamera/request.h>
> +
> +#include <libipa/ipa_interface_wrapper.h>
> +
> +#include "libcamera/internal/buffer.h"
> +#include "libcamera/internal/log.h"
> +
> +namespace libcamera {
> +
> +LOG_DEFINE_CATEGORY(IPAIPU3)
> +
> +class IPAIPU3 : public IPAInterface
> +{
> +public:
> + int init([[maybe_unused]] const IPASettings &settings) override
> + {
> + return 0;
> + }
> + int start([[maybe_unused]] const IPAOperationData &data,
> + [[maybe_unused]] IPAOperationData *result) override { return 0; }
> + void stop() override {}
> +
> + void configure(const CameraSensorInfo &info,
> + const std::map<unsigned int, IPAStream> &streamConfig,
> + const std::map<unsigned int, const ControlInfoMap &> &entityControls,
> + const IPAOperationData &ipaConfig,
> + IPAOperationData *response) override;
> + void mapBuffers(const std::vector<IPABuffer> &buffers) override;
> + void unmapBuffers(const std::vector<unsigned int> &ids) override;
> + void processEvent(const IPAOperationData &event) override;
> +
> +private:
> + void processControls(unsigned int frame, const ControlList &controls);
> + void fillParams(unsigned int frame, ipu3_uapi_params *params);
> + void parseStatistics(unsigned int frame,
> + const ipu3_uapi_stats_3a *stats);
> +
> + void setControls(unsigned int frame);
> +
> + std::map<unsigned int, MappedFrameBuffer> buffers_;
> +
> + ControlInfoMap ctrls_;
> +
> + /* Camera sensor controls. */
> + uint32_t exposure_;
> + uint32_t minExposure_;
> + uint32_t maxExposure_;
> + uint32_t gain_;
> + uint32_t minGain_;
> + uint32_t maxGain_;
> +};
> +
> +void IPAIPU3::configure([[maybe_unused]] const CameraSensorInfo &info,
> + [[maybe_unused]] const std::map<unsigned int, IPAStream> &streamConfig,
> + const std::map<unsigned int, const ControlInfoMap &> &entityControls,
> + [[maybe_unused]] const IPAOperationData &ipaConfig,
> + [[maybe_unused]] IPAOperationData *result)
> +{
> + if (entityControls.empty())
> + return;
> +
> + ctrls_ = entityControls.at(0);
> +
> + const auto itExp = ctrls_.find(V4L2_CID_EXPOSURE);
> + if (itExp == ctrls_.end()) {
> + LOG(IPAIPU3, Error) << "Can't find exposure control";
> + return;
> + }
> +
> + const auto itGain = ctrls_.find(V4L2_CID_ANALOGUE_GAIN);
> + if (itGain == ctrls_.end()) {
> + LOG(IPAIPU3, Error) << "Can't find gain control";
> + return;
> + }
> +
> + minExposure_ = std::max(itExp->second.min().get<int32_t>(), 1);
> + maxExposure_ = itExp->second.max().get<int32_t>();
> + exposure_ = maxExposure_;
> +
> + minGain_ = std::max(itGain->second.min().get<int32_t>(), 1);
> + maxGain_ = itGain->second.max().get<int32_t>();
> + gain_ = maxGain_;
> +
> + setControls(0);
> +}
> +
> +void IPAIPU3::mapBuffers(const std::vector<IPABuffer> &buffers)
> +{
> + for (const IPABuffer &buffer : buffers) {
> + const FrameBuffer fb(buffer.planes);
> + buffers_.emplace(buffer.id,
> + MappedFrameBuffer(&fb, PROT_READ | PROT_WRITE));
> + }
> +}
> +
> +void IPAIPU3::unmapBuffers(const std::vector<unsigned int> &ids)
> +{
> + for (unsigned int id : ids) {
> + auto it = buffers_.find(id);
> + if (it == buffers_.end())
> + continue;
> +
> + buffers_.erase(it);
> + }
> +}
> +
> +void IPAIPU3::processEvent(const IPAOperationData &event)
> +{
> + switch (event.operation) {
> + case IPU3_IPA_EVENT_PROCESS_CONTROLS: {
> + unsigned int frame = event.data[0];
> + processControls(frame, event.controls[0]);
> + break;
> + }
> + case IPU3_IPA_EVENT_STAT_READY: {
> + unsigned int frame = event.data[0];
> + unsigned int bufferId = event.data[1];
> +
> + auto it = buffers_.find(bufferId);
> + if (it == buffers_.end()) {
> + LOG(IPAIPU3, Error) << "Could not find stats buffer!";
> + return;
> + }
> +
> + Span<uint8_t> mem = it->second.maps()[0];
> + const ipu3_uapi_stats_3a *stats =
> + reinterpret_cast<ipu3_uapi_stats_3a *>(mem.data());
> +
> + parseStatistics(frame, stats);
> + break;
> + }
> + case IPU3_IPA_EVENT_FILL_PARAMS: {
> + unsigned int frame = event.data[0];
> + unsigned int bufferId = event.data[1];
> +
> + auto it = buffers_.find(bufferId);
> + if (it == buffers_.end()) {
> + LOG(IPAIPU3, Error) << "Could not find param buffer!";
> + return;
> + }
> +
> + Span<uint8_t> mem = it->second.maps()[0];
> + ipu3_uapi_params *params =
> + reinterpret_cast<ipu3_uapi_params *>(mem.data());
> +
> + fillParams(frame, params);
> + break;
> + }
> + default:
> + LOG(IPAIPU3, Error) << "Unknown event " << event.operation;
> + break;
> + }
> +}
> +
> +void IPAIPU3::processControls([[maybe_unused]] unsigned int frame,
> + [[maybe_unused]] const ControlList &controls)
> +{
> + /* \todo Start processing for 'frame' based on 'controls'. */
> +}
> +
> +void IPAIPU3::fillParams(unsigned int frame, ipu3_uapi_params *params)
> +{
> + /* Prepare parameters buffer. */
> + memset(params, 0, sizeof(*params));
> +
> + /* \todo Fill in parameters buffer. */
> +
> + IPAOperationData op;
> + op.operation = IPU3_IPA_ACTION_PARAM_FILLED;
> +
> + queueFrameAction.emit(frame, op);
> +
> + /* \todo Calculate new values for exposure_ and gain_. */
> + setControls(frame);
> +}
> +
> +void IPAIPU3::parseStatistics(unsigned int frame,
> + [[maybe_unused]] const ipu3_uapi_stats_3a *stats)
> +{
> + ControlList ctrls(controls::controls);
> +
> + /* \todo React to statistics and update internal state machine. */
> + /* \todo Add meta-data information to ctrls. */
> +
> + IPAOperationData op;
> + op.operation = IPU3_IPA_ACTION_METADATA_READY;
> + op.controls.push_back(ctrls);
> +
> + queueFrameAction.emit(frame, op);
> +}
> +
> +void IPAIPU3::setControls(unsigned int frame)
> +{
> + IPAOperationData op;
> + op.operation = IPU3_IPA_ACTION_SET_SENSOR_CONTROLS;
> +
> + ControlList ctrls(ctrls_);
> + ctrls.set(V4L2_CID_EXPOSURE, static_cast<int32_t>(exposure_));
> + ctrls.set(V4L2_CID_ANALOGUE_GAIN, static_cast<int32_t>(gain_));
> + op.controls.push_back(ctrls);
> +
> + queueFrameAction.emit(frame, op);
> +}
> +
> +/*
> + * External IPA module interface
> + */
> +
> +extern "C" {
> +const struct IPAModuleInfo ipaModuleInfo = {
> + IPA_MODULE_API_VERSION,
> + 1,
> + "PipelineHandlerIPU3",
> + "ipu3",
> +};
> +
> +struct ipa_context *ipaCreate()
> +{
> + return new IPAInterfaceWrapper(std::make_unique<IPAIPU3>());
> +}
> +}
> +
> +} /* namespace libcamera */
> diff --git a/src/ipa/ipu3/meson.build b/src/ipa/ipu3/meson.build
> new file mode 100644
> index 0000000000000000..444c82453eac42ff
> --- /dev/null
> +++ b/src/ipa/ipu3/meson.build
> @@ -0,0 +1,21 @@
> +# SPDX-License-Identifier: CC0-1.0
> +
> +ipa_name = 'ipa_ipu3'
> +
> +mod = shared_module(ipa_name,
> + 'ipu3.cpp',
> + name_prefix : '',
> + include_directories : [ ipa_includes, libipa_includes ],
> + dependencies : [ libatomic, libcamera_dep ],
> + link_with : libipa,
> + install : true,
> + install_dir : ipa_install_dir)
> +
> +if ipa_sign_module
> + custom_target(ipa_name + '.so.sign',
> + input : mod,
> + output : ipa_name + '.so.sign',
> + command : [ ipa_sign, ipa_priv_key, '@INPUT@', '@OUTPUT@' ],
> + install : false,
> + build_by_default : true)
> +endif
> diff --git a/src/ipa/meson.build b/src/ipa/meson.build
> index 5a5de267c1477d24..9d623f227a1f9feb 100644
> --- a/src/ipa/meson.build
> +++ b/src/ipa/meson.build
> @@ -19,7 +19,7 @@ subdir('libipa')
>
> ipa_sign = files('ipa-sign.sh')
>
> -ipas = ['raspberrypi', 'rkisp1', 'vimc']
> +ipas = ['ipu3', 'raspberrypi', 'rkisp1', 'vimc']
> ipa_names = []
>
> foreach pipeline : get_option('pipelines')
--
Regards,
Laurent Pinchart
More information about the libcamera-devel
mailing list